2024-04-02T04:37:41.879 INFO:root:teuthology version: 0.0.1.dev262+ge691533 2024-04-02T04:37:41.880 DEBUG:teuthology.run:Teuthology command: teuthology --verbose --name teuthology-2024-04-01_21:08:08-orch-squid-distro-default-smithi --description orch/cephadm/mds_upgrade_sequence/{bluestore-bitmap centos_9.stream conf/{client mds mgr mon osd} fail_fs/no overrides/{ignorelist_health ignorelist_upgrade ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/quincy 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/yes 3-inline/yes 4-verify} 2-client/kclient 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}} --owner scheduled_teuthology@teuthology --archive /home/teuthworker/archive/teuthology-2024-04-01_21:08:08-orch-squid-distro-default-smithi/7634868 -- /home/teuthworker/archive/teuthology-2024-04-01_21:08:08-orch-squid-distro-default-smithi/7634868/orig.config.yaml 2024-04-02T04:37:41.930 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-02T04:37:42.019 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-01_21:08:08-orch-squid-distro-default-smithi/7634868 branch: squid description: orch/cephadm/mds_upgrade_sequence/{bluestore-bitmap centos_9.stream conf/{client mds mgr mon osd} fail_fs/no overrides/{ignorelist_health ignorelist_upgrade ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/quincy 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/yes 3-inline/yes 4-verify} 2-client/kclient 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}} email: ceph-qa@ceph.com first_in_suite: false job_id: '7634868' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'setup ceph/quincy ' name: teuthology-2024-04-01_21:08:08-orch-squid-distro-default-smithi no_nested_subset: false nuke-on-error: true os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: client: client mount timeout: 600 debug client: 20 debug ms: 1 rados mon op timeout: 900 rados osd op timeout: 900 global: mon pg warn min per osd: 0 mds: debug mds: 20 debug mds balancer: 20 debug ms: 1 mds debug frag: true mds debug scatterstat: true mds op complaint time: 180 mds verify scatter: true osd op complaint time: 180 rados mon op timeout: 900 rados osd op timeout: 900 mgr: debug client: 20 debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 mon down mkfs grace: 300 mon op complaint time: 120 osd: bdev async discard: true bdev enable discard: true bluestore allocator: bitmap bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore osd op complaint time: 180 flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - FS_DEGRADED - FS_INLINE_DATA_DEPRECATED - FS_WITH_FAILED_MDS - MDS_ALL_DOWN - MDS_DAMAGE - MDS_DEGRADED - MDS_FAILED - MDS_INSUFFICIENT_STANDBY - MDS_UP_LESS_THAN_MAX - POOL_APP_NOT_ENABLED - overall HEALTH_ - Replacing daemon - OSD_DOWN - overall HEALTH_ - \(OSD_DOWN\) - \(OSD_ - but it is still running - is not responding sha1: 6cec793d9073f0594e2364b9b687ecaf22424cca ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} osd: bdev async discard: true bdev enable discard: true bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs install: ceph: flavor: default sha1: 6cec793d9073f0594e2364b9b687ecaf22424cca kclient: syntax: v1 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: squid sha1: 6cec793d9073f0594e2364b9b687ecaf22424cca owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - host.a - client.0 - osd.0 - osd.1 - osd.2 - - host.b - client.1 - osd.3 - osd.4 - osd.5 seed: 6530 sha1: 6cec793d9073f0594e2364b9b687ecaf22424cca sleep_before_teardown: 0 subset: 52/64 suite: orch suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_6cec793d9073f0594e2364b9b687ecaf22424cca/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 6cec793d9073f0594e2364b9b687ecaf22424cca targets: smithi152.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNs89VekBAWT1tsRx7VcETii7zxD/fJu9Lc5FId3SglEtw2oSgvbzII5Orxu8ecEYhOUtWTLYK2fFOGraHnh0pc= smithi154.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPvKjrTqrf6fGLYuOm9mwzvxtVbhZ/WZTnQYUXal5Epre/f74SKBUFKNQ3Sx5GdJTwjE5x51zT9Hq4F4Br3HI00= tasks: - install: branch: quincy exclude_packages: - ceph-volume - print: '**** done install task...' - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy roleless: true - print: '**** done end installing quincy cephadm ...' - cephadm.shell: host.a: - ceph config set mgr mgr/cephadm/use_repo_digest true --force - print: '**** done cephadm.shell ceph config set mgr...' - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - cephadm.shell: host.a: - ceph fs volume create cephfs --placement=4 - ceph fs dump - cephadm.shell: host.a: - ceph fs set cephfs max_mds 2 - cephadm.shell: host.a: - ceph fs set cephfs allow_standby_replay true - cephadm.shell: host.a: - ceph fs set cephfs inline_data true --yes-i-really-really-mean-it - cephadm.shell: host.a: - ceph fs dump - ceph --format=json fs dump | jq -e ".filesystems | length == 1" - while ! ceph --format=json mds versions | jq -e ". | add == 4"; do sleep 1; done - fs.pre_upgrade_save: null - kclient: null - print: '**** done client' - parallel: - upgrade-tasks - workload-tasks - cephadm.shell: host.a: - ceph fs dump - fs.post_upgrade_checks: null teuthology: fragments_dropped: - /home/teuthology/src/git.ceph.com_ceph_squid/qa/suites/orch/cephadm/mds_upgrade_sequence/tasks/3-upgrade-mgr-staggered.yaml meta: {} postmerge: [] variables: fail_fs: false teuthology_branch: main teuthology_sha1: e691533f9cbb33d85b2187bba20d7102f098636d timestamp: 2024-04-01_21:08:08 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mgr mgr/orchestrator/fail_fs false || true - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph fs dump; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 workload-tasks: sequential: - workunit: clients: all: - suites/fsstress.sh 2024-04-02T04:37:42.019 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_6cec793d9073f0594e2364b9b687ecaf22424cca/qa; will attempt to use it 2024-04-02T04:37:42.022 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_6cec793d9073f0594e2364b9b687ecaf22424cca/qa/tasks 2024-04-02T04:37:42.022 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-02T04:37:42.024 INFO:teuthology.task.internal:Checking packages... 2024-04-02T04:37:42.047 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '6cec793d9073f0594e2364b9b687ecaf22424cca' 2024-04-02T04:37:42.047 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-02T04:37:42.047 INFO:teuthology.packaging:ref: None 2024-04-02T04:37:42.047 INFO:teuthology.packaging:tag: None 2024-04-02T04:37:42.047 INFO:teuthology.packaging:branch: squid 2024-04-02T04:37:42.047 INFO:teuthology.packaging:sha1: 6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T04:37:42.047 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-04-02T04:37:42.305 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-1979.g6cec793d 2024-04-02T04:37:42.307 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-02T04:37:42.313 INFO:teuthology.task.internal:no buildpackages task found 2024-04-02T04:37:42.313 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-02T04:37:42.359 INFO:teuthology.task.internal:Saving configuration 2024-04-02T04:37:42.377 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-02T04:37:42.383 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-02T04:37:42.407 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi152.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-01_21:08:08-orch-squid-distro-default-smithi/7634868', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-02 04:32:23.705768', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNs89VekBAWT1tsRx7VcETii7zxD/fJu9Lc5FId3SglEtw2oSgvbzII5Orxu8ecEYhOUtWTLYK2fFOGraHnh0pc='} 2024-04-02T04:37:42.430 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi154.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-01_21:08:08-orch-squid-distro-default-smithi/7634868', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-02 04:32:23.704322', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPvKjrTqrf6fGLYuOm9mwzvxtVbhZ/WZTnQYUXal5Epre/f74SKBUFKNQ3Sx5GdJTwjE5x51zT9Hq4F4Br3HI00='} 2024-04-02T04:37:42.430 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-02T04:37:42.437 INFO:teuthology.task.internal:roles: ubuntu@smithi152.front.sepia.ceph.com - ['host.a', 'client.0', 'osd.0', 'osd.1', 'osd.2'] 2024-04-02T04:37:42.437 INFO:teuthology.task.internal:roles: ubuntu@smithi154.front.sepia.ceph.com - ['host.b', 'client.1', 'osd.3', 'osd.4', 'osd.5'] 2024-04-02T04:37:42.437 INFO:teuthology.run_tasks:Running task console_log... 2024-04-02T04:37:42.498 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f60ab6cc0d0>, signals=[15]) 2024-04-02T04:37:42.499 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-02T04:37:42.505 INFO:teuthology.task.internal:Opening connections... 2024-04-02T04:37:42.505 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi152.front.sepia.ceph.com 2024-04-02T04:37:42.507 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi152.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T04:37:42.598 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi154.front.sepia.ceph.com 2024-04-02T04:37:42.599 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi154.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T04:37:42.686 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-02T04:37:42.692 DEBUG:teuthology.orchestra.run.smithi152:> uname -m 2024-04-02T04:37:42.709 INFO:teuthology.orchestra.run.smithi152.stdout:x86_64 2024-04-02T04:37:42.709 DEBUG:teuthology.orchestra.run.smithi152:> cat /etc/os-release 2024-04-02T04:37:42.766 INFO:teuthology.orchestra.run.smithi152.stdout:NAME="CentOS Stream" 2024-04-02T04:37:42.766 INFO:teuthology.orchestra.run.smithi152.stdout:VERSION="9" 2024-04-02T04:37:42.766 INFO:teuthology.orchestra.run.smithi152.stdout:ID="centos" 2024-04-02T04:37:42.766 INFO:teuthology.orchestra.run.smithi152.stdout:ID_LIKE="rhel fedora" 2024-04-02T04:37:42.766 INFO:teuthology.orchestra.run.smithi152.stdout:VERSION_ID="9" 2024-04-02T04:37:42.766 INFO:teuthology.orchestra.run.smithi152.stdout:PLATFORM_ID="platform:el9" 2024-04-02T04:37:42.766 INFO:teuthology.orchestra.run.smithi152.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-02T04:37:42.766 INFO:teuthology.orchestra.run.smithi152.stdout:ANSI_COLOR="0;31" 2024-04-02T04:37:42.766 INFO:teuthology.orchestra.run.smithi152.stdout:LOGO="fedora-logo-icon" 2024-04-02T04:37:42.766 INFO:teuthology.orchestra.run.smithi152.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-02T04:37:42.766 INFO:teuthology.orchestra.run.smithi152.stdout:HOME_URL="https://centos.org/" 2024-04-02T04:37:42.767 INFO:teuthology.orchestra.run.smithi152.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-02T04:37:42.767 INFO:teuthology.orchestra.run.smithi152.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-02T04:37:42.767 INFO:teuthology.orchestra.run.smithi152.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-02T04:37:42.767 INFO:teuthology.lock.ops:Updating smithi152.front.sepia.ceph.com on lock server 2024-04-02T04:37:42.790 DEBUG:teuthology.orchestra.run.smithi154:> uname -m 2024-04-02T04:37:42.806 INFO:teuthology.orchestra.run.smithi154.stdout:x86_64 2024-04-02T04:37:42.807 DEBUG:teuthology.orchestra.run.smithi154:> cat /etc/os-release 2024-04-02T04:37:42.863 INFO:teuthology.orchestra.run.smithi154.stdout:NAME="CentOS Stream" 2024-04-02T04:37:42.863 INFO:teuthology.orchestra.run.smithi154.stdout:VERSION="9" 2024-04-02T04:37:42.863 INFO:teuthology.orchestra.run.smithi154.stdout:ID="centos" 2024-04-02T04:37:42.863 INFO:teuthology.orchestra.run.smithi154.stdout:ID_LIKE="rhel fedora" 2024-04-02T04:37:42.863 INFO:teuthology.orchestra.run.smithi154.stdout:VERSION_ID="9" 2024-04-02T04:37:42.863 INFO:teuthology.orchestra.run.smithi154.stdout:PLATFORM_ID="platform:el9" 2024-04-02T04:37:42.864 INFO:teuthology.orchestra.run.smithi154.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-02T04:37:42.864 INFO:teuthology.orchestra.run.smithi154.stdout:ANSI_COLOR="0;31" 2024-04-02T04:37:42.864 INFO:teuthology.orchestra.run.smithi154.stdout:LOGO="fedora-logo-icon" 2024-04-02T04:37:42.864 INFO:teuthology.orchestra.run.smithi154.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-02T04:37:42.864 INFO:teuthology.orchestra.run.smithi154.stdout:HOME_URL="https://centos.org/" 2024-04-02T04:37:42.864 INFO:teuthology.orchestra.run.smithi154.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-02T04:37:42.864 INFO:teuthology.orchestra.run.smithi154.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-02T04:37:42.864 INFO:teuthology.orchestra.run.smithi154.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-02T04:37:42.864 INFO:teuthology.lock.ops:Updating smithi154.front.sepia.ceph.com on lock server 2024-04-02T04:37:42.889 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-02T04:37:42.898 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-02T04:37:42.903 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-02T04:37:42.903 DEBUG:teuthology.orchestra.run.smithi152:> test '!' -e /home/ubuntu/cephtest 2024-04-02T04:37:42.905 DEBUG:teuthology.orchestra.run.smithi154:> test '!' -e /home/ubuntu/cephtest 2024-04-02T04:37:42.920 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-02T04:37:42.926 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-02T04:37:42.926 DEBUG:teuthology.orchestra.run.smithi152:> test -z $(ls -A /var/lib/ceph) 2024-04-02T04:37:42.961 DEBUG:teuthology.orchestra.run.smithi154:> test -z $(ls -A /var/lib/ceph) 2024-04-02T04:37:42.989 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-02T04:37:43.047 INFO:teuthology.run_tasks:Running task kernel... 2024-04-02T04:37:43.063 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2024-04-02T04:37:43.064 INFO:teuthology.task.kernel:config {'host.a': {'kdb': True, 'sha1': 'distro'}, 'host.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2024-04-02T04:37:43.064 DEBUG:teuthology.orchestra.run.smithi152:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-02T04:37:43.064 DEBUG:teuthology.orchestra.run.smithi154:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-02T04:37:43.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T04:37:43.080 DEBUG:teuthology.orchestra.run.smithi154:> uname -r 2024-04-02T04:37:43.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T04:37:43.081 DEBUG:teuthology.orchestra.run.smithi152:> uname -r 2024-04-02T04:37:43.135 INFO:teuthology.orchestra.run.smithi154.stdout:5.14.0-361.el9.x86_64 2024-04-02T04:37:43.135 INFO:teuthology.task.kernel:Running kernel on smithi154: 5.14.0-361.el9.x86_64 2024-04-02T04:37:43.135 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum install -y kernel 2024-04-02T04:37:43.138 INFO:teuthology.orchestra.run.smithi152.stdout:5.14.0-361.el9.x86_64 2024-04-02T04:37:43.138 INFO:teuthology.task.kernel:Running kernel on smithi152: 5.14.0-361.el9.x86_64 2024-04-02T04:37:43.138 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum install -y kernel 2024-04-02T04:37:45.087 INFO:teuthology.orchestra.run.smithi154.stdout:CentOS-9 - AppStream 17 MB/s | 6.7 MB 00:00 2024-04-02T04:37:45.276 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS-9 - AppStream 14 MB/s | 6.7 MB 00:00 2024-04-02T04:37:46.693 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS-9 - BaseOS 6.9 MB/s | 2.0 MB 00:00 2024-04-02T04:37:46.712 INFO:teuthology.orchestra.run.smithi154.stdout:CentOS-9 - BaseOS 3.9 MB/s | 2.0 MB 00:00 2024-04-02T04:37:50.142 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - BaseOS 2.6 MB/s | 8.0 MB 00:03 2024-04-02T04:37:52.826 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - AppStream 12 MB/s | 19 MB 00:01 2024-04-02T04:37:56.819 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - CRB 13 MB/s | 6.1 MB 00:00 2024-04-02T04:37:58.210 INFO:teuthology.orchestra.run.smithi154.stdout:CentOS Stream 9 - BaseOS 738 kB/s | 8.0 MB 00:11 2024-04-02T04:37:58.361 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - Extras packages 39 kB/s | 16 kB 00:00 2024-04-02T04:38:00.479 INFO:teuthology.orchestra.run.smithi152.stdout:Extra Packages for Enterprise Linux 10 MB/s | 21 MB 00:01 2024-04-02T04:38:00.935 INFO:teuthology.orchestra.run.smithi154.stdout:CentOS Stream 9 - AppStream 11 MB/s | 19 MB 00:01 2024-04-02T04:38:05.387 INFO:teuthology.orchestra.run.smithi152.stdout:lab-extras 34 kB/s | 1.7 kB 00:00 2024-04-02T04:38:05.569 INFO:teuthology.orchestra.run.smithi154.stdout:CentOS Stream 9 - CRB 5.4 MB/s | 6.1 MB 00:01 2024-04-02T04:38:07.053 INFO:teuthology.orchestra.run.smithi154.stdout:CentOS Stream 9 - Extras packages 47 kB/s | 16 kB 00:00 2024-04-02T04:38:07.636 INFO:teuthology.orchestra.run.smithi152.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-02T04:38:07.636 INFO:teuthology.orchestra.run.smithi152.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-02T04:38:07.753 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-02T04:38:07.764 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:38:07.764 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-02T04:38:07.764 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:38:07.764 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-02T04:38:07.764 INFO:teuthology.orchestra.run.smithi152.stdout: kernel x86_64 5.14.0-434.el9 CentOS-BaseOS 192 k 2024-04-02T04:38:07.764 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-02T04:38:07.764 INFO:teuthology.orchestra.run.smithi152.stdout: kernel-core x86_64 5.14.0-434.el9 CentOS-BaseOS 15 M 2024-04-02T04:38:07.764 INFO:teuthology.orchestra.run.smithi152.stdout: kernel-modules x86_64 5.14.0-434.el9 CentOS-BaseOS 34 M 2024-04-02T04:38:07.765 INFO:teuthology.orchestra.run.smithi152.stdout: kernel-modules-core x86_64 5.14.0-434.el9 CentOS-BaseOS 28 M 2024-04-02T04:38:07.765 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:38:07.765 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-02T04:38:07.765 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:38:07.765 INFO:teuthology.orchestra.run.smithi152.stdout:Install 4 Packages 2024-04-02T04:38:07.765 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:38:07.766 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 76 M 2024-04-02T04:38:07.766 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 123 M 2024-04-02T04:38:07.766 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-02T04:38:07.914 INFO:teuthology.orchestra.run.smithi152.stdout:(1/4): kernel-5.14.0-434.el9.x86_64.rpm 1.3 MB/s | 192 kB 00:00 2024-04-02T04:38:08.414 INFO:teuthology.orchestra.run.smithi152.stdout:(2/4): kernel-core-5.14.0-434.el9.x86_64.rpm 23 MB/s | 15 MB 00:00 2024-04-02T04:38:08.832 INFO:teuthology.orchestra.run.smithi154.stdout:Extra Packages for Enterprise Linux 13 MB/s | 21 MB 00:01 2024-04-02T04:38:08.964 INFO:teuthology.orchestra.run.smithi152.stdout:(3/4): kernel-modules-5.14.0-434.el9.x86_64.rpm 28 MB/s | 34 MB 00:01 2024-04-02T04:38:09.356 INFO:teuthology.orchestra.run.smithi152.stdout:(4/4): kernel-modules-core-5.14.0-434.el9.x86_6 19 MB/s | 28 MB 00:01 2024-04-02T04:38:09.360 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:38:09.360 INFO:teuthology.orchestra.run.smithi152.stdout:Total 48 MB/s | 76 MB 00:01 2024-04-02T04:38:09.360 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-02T04:38:09.734 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-02T04:38:09.734 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-02T04:38:10.554 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-02T04:38:10.554 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-02T04:38:12.857 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-02T04:38:13.458 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : kernel-modules-core-5.14.0-434.el9.x86_64 1/4 2024-04-02T04:38:13.544 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T04:38:13.804 INFO:teuthology.orchestra.run.smithi154.stdout:lab-extras 41 kB/s | 1.7 kB 00:00 2024-04-02T04:38:16.203 INFO:teuthology.orchestra.run.smithi154.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-02T04:38:16.203 INFO:teuthology.orchestra.run.smithi154.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-04-02T04:38:16.286 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T04:38:16.291 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-04-02T04:38:16.295 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:38:16.295 INFO:teuthology.orchestra.run.smithi154.stdout: Package Arch Version Repository Size 2024-04-02T04:38:16.295 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:38:16.295 INFO:teuthology.orchestra.run.smithi154.stdout:Installing: 2024-04-02T04:38:16.296 INFO:teuthology.orchestra.run.smithi154.stdout: kernel x86_64 5.14.0-434.el9 CentOS-BaseOS 192 k 2024-04-02T04:38:16.296 INFO:teuthology.orchestra.run.smithi154.stdout:Installing dependencies: 2024-04-02T04:38:16.296 INFO:teuthology.orchestra.run.smithi154.stdout: kernel-core x86_64 5.14.0-434.el9 CentOS-BaseOS 15 M 2024-04-02T04:38:16.296 INFO:teuthology.orchestra.run.smithi154.stdout: kernel-modules x86_64 5.14.0-434.el9 CentOS-BaseOS 34 M 2024-04-02T04:38:16.296 INFO:teuthology.orchestra.run.smithi154.stdout: kernel-modules-core x86_64 5.14.0-434.el9 CentOS-BaseOS 28 M 2024-04-02T04:38:16.296 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:38:16.296 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction Summary 2024-04-02T04:38:16.296 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:38:16.296 INFO:teuthology.orchestra.run.smithi154.stdout:Install 4 Packages 2024-04-02T04:38:16.296 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:38:16.297 INFO:teuthology.orchestra.run.smithi154.stdout:Total download size: 76 M 2024-04-02T04:38:16.297 INFO:teuthology.orchestra.run.smithi154.stdout:Installed size: 123 M 2024-04-02T04:38:16.297 INFO:teuthology.orchestra.run.smithi154.stdout:Downloading Packages: 2024-04-02T04:38:16.432 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T04:38:16.456 INFO:teuthology.orchestra.run.smithi154.stdout:(1/4): kernel-5.14.0-434.el9.x86_64.rpm 1.2 MB/s | 192 kB 00:00 2024-04-02T04:38:16.914 INFO:teuthology.orchestra.run.smithi154.stdout:(2/4): kernel-core-5.14.0-434.el9.x86_64.rpm 24 MB/s | 15 MB 00:00 2024-04-02T04:38:17.417 INFO:teuthology.orchestra.run.smithi154.stdout:(3/4): kernel-modules-5.14.0-434.el9.x86_64.rpm 30 MB/s | 34 MB 00:01 2024-04-02T04:38:17.873 INFO:teuthology.orchestra.run.smithi154.stdout:(4/4): kernel-modules-core-5.14.0-434.el9.x86_6 20 MB/s | 28 MB 00:01 2024-04-02T04:38:17.876 INFO:teuthology.orchestra.run.smithi154.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:38:17.876 INFO:teuthology.orchestra.run.smithi154.stdout:Total 48 MB/s | 76 MB 00:01 2024-04-02T04:38:17.877 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction check 2024-04-02T04:38:18.256 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction check succeeded. 2024-04-02T04:38:18.256 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction test 2024-04-02T04:38:19.157 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction test succeeded. 2024-04-02T04:38:19.157 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction 2024-04-02T04:38:21.155 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T04:38:21.197 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : kernel-5.14.0-434.el9.x86_64 4/4 2024-04-02T04:38:21.473 INFO:teuthology.orchestra.run.smithi154.stdout: Preparing : 1/1 2024-04-02T04:38:22.082 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : kernel-modules-core-5.14.0-434.el9.x86_64 1/4 2024-04-02T04:38:22.166 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T04:38:24.951 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T04:38:25.087 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T04:38:25.959 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T04:38:29.853 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T04:38:29.919 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : kernel-5.14.0-434.el9.x86_64 4/4 2024-04-02T04:38:34.856 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T04:38:55.149 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T04:38:55.150 INFO:teuthology.orchestra.run.smithi152.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-02T04:38:55.150 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:38:55.193 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 4/4 2024-04-02T04:38:55.547 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: kernel-5.14.0-434.el9.x86_64 4/4 2024-04-02T04:38:55.547 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : kernel-5.14.0-434.el9.x86_64 1/4 2024-04-02T04:38:55.547 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T04:38:55.547 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T04:38:55.977 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T04:38:55.977 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:38:55.977 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-02T04:38:55.978 INFO:teuthology.orchestra.run.smithi152.stdout: kernel-5.14.0-434.el9.x86_64 2024-04-02T04:38:55.978 INFO:teuthology.orchestra.run.smithi152.stdout: kernel-core-5.14.0-434.el9.x86_64 2024-04-02T04:38:55.978 INFO:teuthology.orchestra.run.smithi152.stdout: kernel-modules-5.14.0-434.el9.x86_64 2024-04-02T04:38:55.978 INFO:teuthology.orchestra.run.smithi152.stdout: kernel-modules-core-5.14.0-434.el9.x86_64 2024-04-02T04:38:55.978 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:38:55.978 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-02T04:38:56.285 DEBUG:teuthology.orchestra.run.smithi152:> rpm -q kernel | sort -rV | head -n 1 2024-04-02T04:38:56.397 INFO:teuthology.orchestra.run.smithi152.stdout:kernel-5.14.0-434.el9.x86_64 2024-04-02T04:38:56.398 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-04-02T04:38:56.398 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-434.el9.x86_64 2024-04-02T04:38:56.398 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-02T04:38:56.398 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-02T04:38:56.398 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2024-04-02T04:38:56.398 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi152.front.sepia.ceph.com, path=None, version=distro) 2024-04-02T04:38:56.398 DEBUG:teuthology.orchestra.run.smithi152:> rpm -q kernel | sort -rV | head -n 1 2024-04-02T04:38:56.479 INFO:teuthology.orchestra.run.smithi152.stdout:kernel-5.14.0-434.el9.x86_64 2024-04-02T04:38:56.480 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-04-02T04:38:56.480 DEBUG:teuthology.orchestra.run.smithi152:> sudo rpm -qi grub2-tools 2024-04-02T04:38:56.523 INFO:teuthology.orchestra.run.smithi152.stdout:Name : grub2-tools 2024-04-02T04:38:56.523 INFO:teuthology.orchestra.run.smithi152.stdout:Epoch : 1 2024-04-02T04:38:56.524 INFO:teuthology.orchestra.run.smithi152.stdout:Version : 2.06 2024-04-02T04:38:56.524 INFO:teuthology.orchestra.run.smithi152.stdout:Release : 68.el9 2024-04-02T04:38:56.524 INFO:teuthology.orchestra.run.smithi152.stdout:Architecture: x86_64 2024-04-02T04:38:56.524 INFO:teuthology.orchestra.run.smithi152.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-02T04:38:56.524 INFO:teuthology.orchestra.run.smithi152.stdout:Group : Unspecified 2024-04-02T04:38:56.524 INFO:teuthology.orchestra.run.smithi152.stdout:Size : 8264254 2024-04-02T04:38:56.524 INFO:teuthology.orchestra.run.smithi152.stdout:License : GPLv3+ 2024-04-02T04:38:56.524 INFO:teuthology.orchestra.run.smithi152.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-02T04:38:56.524 INFO:teuthology.orchestra.run.smithi152.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-02T04:38:56.524 INFO:teuthology.orchestra.run.smithi152.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-02T04:38:56.524 INFO:teuthology.orchestra.run.smithi152.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-02T04:38:56.524 INFO:teuthology.orchestra.run.smithi152.stdout:Packager : builder@centos.org 2024-04-02T04:38:56.524 INFO:teuthology.orchestra.run.smithi152.stdout:Vendor : CentOS 2024-04-02T04:38:56.524 INFO:teuthology.orchestra.run.smithi152.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-02T04:38:56.524 INFO:teuthology.orchestra.run.smithi152.stdout:Summary : Support tools for GRUB. 2024-04-02T04:38:56.525 INFO:teuthology.orchestra.run.smithi152.stdout:Description : 2024-04-02T04:38:56.525 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:38:56.525 INFO:teuthology.orchestra.run.smithi152.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-02T04:38:56.525 INFO:teuthology.orchestra.run.smithi152.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-02T04:38:56.525 INFO:teuthology.orchestra.run.smithi152.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-02T04:38:56.525 INFO:teuthology.orchestra.run.smithi152.stdout:hardware devices. 2024-04-02T04:38:56.525 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:38:56.525 INFO:teuthology.orchestra.run.smithi152.stdout:This subpackage provides tools for support of all platforms. 2024-04-02T04:38:56.526 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-02T04:38:56.526 INFO:teuthology.task.kernel:Updating grub on smithi152 to boot 5.14.0-434.el9.x86_64 2024-04-02T04:38:56.526 DEBUG:teuthology.orchestra.run.smithi152:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-02T04:38:57.128 INFO:teuthology.orchestra.run.smithi152.stderr:Generating grub configuration file ... 2024-04-02T04:38:58.557 INFO:teuthology.orchestra.run.smithi152.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-02T04:38:58.587 INFO:teuthology.orchestra.run.smithi152.stderr:done 2024-04-02T04:38:58.589 DEBUG:teuthology.orchestra.run.smithi152:> mktemp 2024-04-02T04:38:58.603 INFO:teuthology.orchestra.run.smithi152.stdout:/tmp/tmp.Pq60uR4XrK 2024-04-02T04:38:58.603 DEBUG:teuthology.orchestra.run.smithi152:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.Pq60uR4XrK 2024-04-02T04:38:58.672 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod 0666 /tmp/tmp.Pq60uR4XrK 2024-04-02T04:38:58.808 DEBUG:teuthology.orchestra.remote:smithi152:/tmp/tmp.Pq60uR4XrK is 6KB 2024-04-02T04:38:58.861 DEBUG:teuthology.orchestra.run.smithi152:> rm -fr /tmp/tmp.Pq60uR4XrK 2024-04-02T04:38:58.875 DEBUG:teuthology.orchestra.run.smithi152:> sudo /bin/ls /boot/loader/entries || true 2024-04-02T04:38:58.941 INFO:teuthology.orchestra.run.smithi152.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-02T04:38:58.941 INFO:teuthology.orchestra.run.smithi152.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-02T04:38:58.941 INFO:teuthology.orchestra.run.smithi152.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-02T04:38:58.941 INFO:teuthology.orchestra.run.smithi152.stdout:15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64.conf 2024-04-02T04:38:58.942 DEBUG:teuthology.orchestra.run.smithi152:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64 2024-04-02T04:38:59.108 DEBUG:teuthology.orchestra.run.smithi152:> sudo shutdown -r now 2024-04-02T04:39:04.460 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T04:39:04.461 INFO:teuthology.orchestra.run.smithi154.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-04-02T04:39:04.461 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:39:04.504 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 4/4 2024-04-02T04:39:04.896 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: kernel-5.14.0-434.el9.x86_64 4/4 2024-04-02T04:39:04.896 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : kernel-5.14.0-434.el9.x86_64 1/4 2024-04-02T04:39:04.896 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-04-02T04:39:04.896 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-04-02T04:39:05.338 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-04-02T04:39:05.338 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:39:05.338 INFO:teuthology.orchestra.run.smithi154.stdout:Installed: 2024-04-02T04:39:05.338 INFO:teuthology.orchestra.run.smithi154.stdout: kernel-5.14.0-434.el9.x86_64 2024-04-02T04:39:05.338 INFO:teuthology.orchestra.run.smithi154.stdout: kernel-core-5.14.0-434.el9.x86_64 2024-04-02T04:39:05.338 INFO:teuthology.orchestra.run.smithi154.stdout: kernel-modules-5.14.0-434.el9.x86_64 2024-04-02T04:39:05.338 INFO:teuthology.orchestra.run.smithi154.stdout: kernel-modules-core-5.14.0-434.el9.x86_64 2024-04-02T04:39:05.338 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:39:05.338 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-04-02T04:39:05.666 DEBUG:teuthology.orchestra.run.smithi154:> rpm -q kernel | sort -rV | head -n 1 2024-04-02T04:39:05.814 INFO:teuthology.orchestra.run.smithi154.stdout:kernel-5.14.0-434.el9.x86_64 2024-04-02T04:39:05.814 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-04-02T04:39:05.814 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-434.el9.x86_64 2024-04-02T04:39:05.815 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-02T04:39:05.815 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-02T04:39:05.815 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2024-04-02T04:39:05.815 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi154.front.sepia.ceph.com, path=None, version=distro) 2024-04-02T04:39:05.815 DEBUG:teuthology.orchestra.run.smithi154:> rpm -q kernel | sort -rV | head -n 1 2024-04-02T04:39:05.895 INFO:teuthology.orchestra.run.smithi154.stdout:kernel-5.14.0-434.el9.x86_64 2024-04-02T04:39:05.895 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-04-02T04:39:05.895 DEBUG:teuthology.orchestra.run.smithi154:> sudo rpm -qi grub2-tools 2024-04-02T04:39:05.929 INFO:teuthology.orchestra.run.smithi154.stdout:Name : grub2-tools 2024-04-02T04:39:05.929 INFO:teuthology.orchestra.run.smithi154.stdout:Epoch : 1 2024-04-02T04:39:05.930 INFO:teuthology.orchestra.run.smithi154.stdout:Version : 2.06 2024-04-02T04:39:05.930 INFO:teuthology.orchestra.run.smithi154.stdout:Release : 68.el9 2024-04-02T04:39:05.930 INFO:teuthology.orchestra.run.smithi154.stdout:Architecture: x86_64 2024-04-02T04:39:05.930 INFO:teuthology.orchestra.run.smithi154.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-04-02T04:39:05.930 INFO:teuthology.orchestra.run.smithi154.stdout:Group : Unspecified 2024-04-02T04:39:05.930 INFO:teuthology.orchestra.run.smithi154.stdout:Size : 8264254 2024-04-02T04:39:05.930 INFO:teuthology.orchestra.run.smithi154.stdout:License : GPLv3+ 2024-04-02T04:39:05.930 INFO:teuthology.orchestra.run.smithi154.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-04-02T04:39:05.930 INFO:teuthology.orchestra.run.smithi154.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-04-02T04:39:05.930 INFO:teuthology.orchestra.run.smithi154.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-04-02T04:39:05.930 INFO:teuthology.orchestra.run.smithi154.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-04-02T04:39:05.930 INFO:teuthology.orchestra.run.smithi154.stdout:Packager : builder@centos.org 2024-04-02T04:39:05.930 INFO:teuthology.orchestra.run.smithi154.stdout:Vendor : CentOS 2024-04-02T04:39:05.931 INFO:teuthology.orchestra.run.smithi154.stdout:URL : http://www.gnu.org/software/grub/ 2024-04-02T04:39:05.931 INFO:teuthology.orchestra.run.smithi154.stdout:Summary : Support tools for GRUB. 2024-04-02T04:39:05.931 INFO:teuthology.orchestra.run.smithi154.stdout:Description : 2024-04-02T04:39:05.931 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:39:05.931 INFO:teuthology.orchestra.run.smithi154.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-04-02T04:39:05.931 INFO:teuthology.orchestra.run.smithi154.stdout:customizable bootloader with modular architecture. It supports a rich 2024-04-02T04:39:05.931 INFO:teuthology.orchestra.run.smithi154.stdout:variety of kernel formats, file systems, computer architectures and 2024-04-02T04:39:05.931 INFO:teuthology.orchestra.run.smithi154.stdout:hardware devices. 2024-04-02T04:39:05.931 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:39:05.931 INFO:teuthology.orchestra.run.smithi154.stdout:This subpackage provides tools for support of all platforms. 2024-04-02T04:39:05.932 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-04-02T04:39:05.932 INFO:teuthology.task.kernel:Updating grub on smithi154 to boot 5.14.0-434.el9.x86_64 2024-04-02T04:39:05.932 DEBUG:teuthology.orchestra.run.smithi154:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-04-02T04:39:06.561 INFO:teuthology.orchestra.run.smithi154.stderr:Generating grub configuration file ... 2024-04-02T04:39:07.933 INFO:teuthology.orchestra.run.smithi154.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-04-02T04:39:07.971 INFO:teuthology.orchestra.run.smithi154.stderr:done 2024-04-02T04:39:07.974 DEBUG:teuthology.orchestra.run.smithi154:> mktemp 2024-04-02T04:39:07.988 INFO:teuthology.orchestra.run.smithi154.stdout:/tmp/tmp.hTuhj0mrhi 2024-04-02T04:39:07.989 DEBUG:teuthology.orchestra.run.smithi154:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.hTuhj0mrhi 2024-04-02T04:39:08.058 DEBUG:teuthology.orchestra.run.smithi154:> sudo chmod 0666 /tmp/tmp.hTuhj0mrhi 2024-04-02T04:39:08.191 DEBUG:teuthology.orchestra.remote:smithi154:/tmp/tmp.hTuhj0mrhi is 6KB 2024-04-02T04:39:08.240 DEBUG:teuthology.orchestra.run.smithi154:> rm -fr /tmp/tmp.hTuhj0mrhi 2024-04-02T04:39:08.254 DEBUG:teuthology.orchestra.run.smithi154:> sudo /bin/ls /boot/loader/entries || true 2024-04-02T04:39:08.318 INFO:teuthology.orchestra.run.smithi154.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-04-02T04:39:08.318 INFO:teuthology.orchestra.run.smithi154.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-04-02T04:39:08.318 INFO:teuthology.orchestra.run.smithi154.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-04-02T04:39:08.319 INFO:teuthology.orchestra.run.smithi154.stdout:15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64.conf 2024-04-02T04:39:08.320 DEBUG:teuthology.orchestra.run.smithi154:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64 2024-04-02T04:39:08.461 DEBUG:teuthology.orchestra.run.smithi154:> sudo shutdown -r now 2024-04-02T04:39:29.130 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2024-04-02T04:39:29.131 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi152.front.sepia.ceph.com' 2024-04-02T04:39:29.131 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi152.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T04:39:38.473 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2024-04-02T04:39:38.473 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi154.front.sepia.ceph.com' 2024-04-02T04:39:38.474 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi154.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T04:39:47.491 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.152 2024-04-02T04:39:56.498 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi152.front.sepia.ceph.com' 2024-04-02T04:39:56.499 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi152.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T04:39:56.958 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.154 2024-04-02T04:39:59.550 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.152 2024-04-02T04:40:05.962 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi154.front.sepia.ceph.com' 2024-04-02T04:40:05.963 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi154.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T04:40:09.022 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.154 2024-04-02T04:40:11.553 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi152.front.sepia.ceph.com' 2024-04-02T04:40:11.554 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi152.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T04:40:21.030 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi154.front.sepia.ceph.com' 2024-04-02T04:40:21.031 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi154.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T04:40:39.459 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.154 2024-04-02T04:40:54.474 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi154.front.sepia.ceph.com' 2024-04-02T04:40:54.475 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi154.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T04:40:54.995 DEBUG:teuthology.orchestra.run.smithi154:> true 2024-04-02T04:40:55.444 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi154.front.sepia.ceph.com' 2024-04-02T04:40:55.444 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "5.14.0-434.el9.x86_64"... 2024-04-02T04:40:55.444 DEBUG:teuthology.orchestra.run.smithi154:> uname -r 2024-04-02T04:40:55.458 INFO:teuthology.orchestra.run.smithi154.stdout:5.14.0-434.el9.x86_64 2024-04-02T04:40:55.458 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-434.el9.x86_64 vs 5.14.0-434.el9.x86_64 2024-04-02T04:40:55.459 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-02T04:40:55.459 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-02T04:40:56.459 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-04-02T04:40:56.460 DEBUG:teuthology.orchestra.run.smithi154:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-02T04:40:56.537 INFO:teuthology.orchestra.run.smithi154.stdout:ttyS1 2024-04-02T04:40:56.581 DEBUG:teuthology.parallel:result is None 2024-04-02T04:41:11.556 DEBUG:teuthology.orchestra.remote:timed out 2024-04-02T04:41:26.557 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi152.front.sepia.ceph.com' 2024-04-02T04:41:26.559 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi152.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T04:41:26.861 DEBUG:teuthology.orchestra.run.smithi152:> true 2024-04-02T04:41:27.257 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi152.front.sepia.ceph.com' 2024-04-02T04:41:27.257 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "5.14.0-434.el9.x86_64"... 2024-04-02T04:41:27.257 DEBUG:teuthology.orchestra.run.smithi152:> uname -r 2024-04-02T04:41:27.274 INFO:teuthology.orchestra.run.smithi152.stdout:5.14.0-434.el9.x86_64 2024-04-02T04:41:27.274 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-434.el9.x86_64 vs 5.14.0-434.el9.x86_64 2024-04-02T04:41:27.274 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-02T04:41:27.274 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-04-02T04:41:28.276 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-04-02T04:41:28.276 DEBUG:teuthology.orchestra.run.smithi152:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-02T04:41:28.360 INFO:teuthology.orchestra.run.smithi152.stdout:ttyS1 2024-04-02T04:41:28.394 DEBUG:teuthology.parallel:result is None 2024-04-02T04:41:28.395 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-02T04:41:28.405 INFO:teuthology.task.internal:Creating test directory... 2024-04-02T04:41:28.405 DEBUG:teuthology.orchestra.run.smithi152:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-02T04:41:28.408 DEBUG:teuthology.orchestra.run.smithi154:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-02T04:41:28.427 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-02T04:41:28.438 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-02T04:41:28.453 INFO:teuthology.task.internal:Creating archive directory... 2024-04-02T04:41:28.453 DEBUG:teuthology.orchestra.run.smithi152:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-02T04:41:28.466 DEBUG:teuthology.orchestra.run.smithi154:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-02T04:41:28.517 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-02T04:41:28.529 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-02T04:41:28.529 DEBUG:teuthology.orchestra.run.smithi152:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-02T04:41:28.532 DEBUG:teuthology.orchestra.run.smithi154:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-02T04:41:28.566 INFO:teuthology.orchestra.run.smithi152.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-02T04:41:28.580 INFO:teuthology.orchestra.run.smithi152.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-02T04:41:28.585 INFO:teuthology.orchestra.run.smithi154.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-02T04:41:28.600 INFO:teuthology.orchestra.run.smithi154.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-02T04:41:28.602 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-02T04:41:28.610 INFO:teuthology.task.internal:Configuring sudo... 2024-04-02T04:41:28.610 DEBUG:teuthology.orchestra.run.smithi152:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-02T04:41:28.624 DEBUG:teuthology.orchestra.run.smithi154:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-02T04:41:28.672 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-02T04:41:28.685 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-02T04:41:28.685 DEBUG:teuthology.orchestra.run.smithi152:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-02T04:41:28.695 DEBUG:teuthology.orchestra.run.smithi154:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-02T04:41:28.728 DEBUG:teuthology.orchestra.run.smithi152:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-02T04:41:28.795 DEBUG:teuthology.orchestra.run.smithi152:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-02T04:41:28.845 DEBUG:teuthology.orchestra.run.smithi152:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-02T04:41:28.928 DEBUG:teuthology.orchestra.run.smithi152:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-02T04:41:28.972 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-02T04:41:28.972 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-02T04:41:29.041 DEBUG:teuthology.orchestra.run.smithi154:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-02T04:41:29.084 DEBUG:teuthology.orchestra.run.smithi154:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-02T04:41:29.136 DEBUG:teuthology.orchestra.run.smithi154:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-02T04:41:29.221 DEBUG:teuthology.orchestra.run.smithi154:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-02T04:41:29.263 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-04-02T04:41:29.264 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-02T04:41:29.332 DEBUG:teuthology.orchestra.run.smithi152:> sudo service rsyslog restart 2024-04-02T04:41:29.334 DEBUG:teuthology.orchestra.run.smithi154:> sudo service rsyslog restart 2024-04-02T04:41:29.379 INFO:teuthology.orchestra.run.smithi152.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-02T04:41:29.419 INFO:teuthology.orchestra.run.smithi154.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-02T04:41:29.789 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-02T04:41:29.799 INFO:teuthology.task.internal:Starting timer... 2024-04-02T04:41:29.799 INFO:teuthology.run_tasks:Running task pcp... 2024-04-02T04:41:29.816 INFO:teuthology.run_tasks:Running task selinux... 2024-04-02T04:41:29.828 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-04-02T04:41:29.829 DEBUG:teuthology.orchestra.run.smithi152:> sudo service auditd rotate 2024-04-02T04:41:29.886 INFO:teuthology.orchestra.run.smithi152.stdout:Rotating logs: 2024-04-02T04:41:29.887 DEBUG:teuthology.orchestra.run.smithi154:> sudo service auditd rotate 2024-04-02T04:41:29.947 INFO:teuthology.orchestra.run.smithi154.stdout:Rotating logs: 2024-04-02T04:41:29.949 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-02T04:41:29.949 DEBUG:teuthology.orchestra.run.smithi152:> /usr/sbin/getenforce 2024-04-02T04:41:29.973 INFO:teuthology.orchestra.run.smithi152.stdout:Permissive 2024-04-02T04:41:29.974 DEBUG:teuthology.orchestra.run.smithi154:> /usr/sbin/getenforce 2024-04-02T04:41:29.999 INFO:teuthology.orchestra.run.smithi154.stdout:Permissive 2024-04-02T04:41:29.999 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi152.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi154.front.sepia.ceph.com': 'permissive'} 2024-04-02T04:41:29.999 DEBUG:teuthology.orchestra.run.smithi152:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-02T04:41:30.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T04:41:30.042 DEBUG:teuthology.orchestra.run.smithi154:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-02T04:41:30.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T04:41:30.071 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-02T04:41:30.071 DEBUG:teuthology.orchestra.run.smithi152:> sudo /usr/sbin/setenforce permissive 2024-04-02T04:41:30.118 DEBUG:teuthology.orchestra.run.smithi154:> sudo /usr/sbin/setenforce permissive 2024-04-02T04:41:30.150 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-02T04:41:30.163 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-02T04:41:30.180 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-02T04:41:30.182 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi152.front.sepia.ceph.com,smithi154.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-02T04:45:47.646 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi152.front.sepia.ceph.com'), Remote(name='ubuntu@smithi154.front.sepia.ceph.com')] 2024-04-02T04:45:47.647 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi152.front.sepia.ceph.com' 2024-04-02T04:45:47.648 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi152.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T04:45:47.743 DEBUG:teuthology.orchestra.run.smithi152:> true 2024-04-02T04:45:47.825 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi152.front.sepia.ceph.com' 2024-04-02T04:45:47.825 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi154.front.sepia.ceph.com' 2024-04-02T04:45:47.826 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi154.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-02T04:45:47.953 DEBUG:teuthology.orchestra.run.smithi154:> true 2024-04-02T04:45:47.990 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi154.front.sepia.ceph.com' 2024-04-02T04:45:47.990 INFO:teuthology.run_tasks:Running task clock... 2024-04-02T04:45:48.000 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-02T04:45:48.001 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-02T04:45:48.001 DEBUG:teuthology.orchestra.run.smithi152:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-02T04:45:48.005 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-02T04:45:48.005 DEBUG:teuthology.orchestra.run.smithi154:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-02T04:45:48.033 INFO:teuthology.orchestra.run.smithi152.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-02T04:45:48.049 INFO:teuthology.orchestra.run.smithi152.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-02T04:45:48.062 INFO:teuthology.orchestra.run.smithi154.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-02T04:45:48.079 INFO:teuthology.orchestra.run.smithi154.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-02T04:45:48.079 INFO:teuthology.orchestra.run.smithi152.stderr:sudo: ntpd: command not found 2024-04-02T04:45:48.092 INFO:teuthology.orchestra.run.smithi152.stdout:506 Cannot talk to daemon 2024-04-02T04:45:48.108 INFO:teuthology.orchestra.run.smithi154.stderr:sudo: ntpd: command not found 2024-04-02T04:45:48.109 INFO:teuthology.orchestra.run.smithi152.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-02T04:45:48.122 INFO:teuthology.orchestra.run.smithi154.stdout:506 Cannot talk to daemon 2024-04-02T04:45:48.125 INFO:teuthology.orchestra.run.smithi152.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-02T04:45:48.140 INFO:teuthology.orchestra.run.smithi154.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-02T04:45:48.155 INFO:teuthology.orchestra.run.smithi154.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-02T04:45:48.179 INFO:teuthology.orchestra.run.smithi152.stderr:bash: line 1: ntpq: command not found 2024-04-02T04:45:48.182 INFO:teuthology.orchestra.run.smithi152.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-02T04:45:48.182 INFO:teuthology.orchestra.run.smithi152.stdout:=============================================================================== 2024-04-02T04:45:48.182 INFO:teuthology.orchestra.run.smithi152.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T04:45:48.182 INFO:teuthology.orchestra.run.smithi152.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T04:45:48.183 INFO:teuthology.orchestra.run.smithi152.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T04:45:48.183 INFO:teuthology.orchestra.run.smithi152.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T04:45:48.211 INFO:teuthology.orchestra.run.smithi154.stderr:bash: line 1: ntpq: command not found 2024-04-02T04:45:48.214 INFO:teuthology.orchestra.run.smithi154.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-02T04:45:48.214 INFO:teuthology.orchestra.run.smithi154.stdout:=============================================================================== 2024-04-02T04:45:48.215 INFO:teuthology.orchestra.run.smithi154.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T04:45:48.215 INFO:teuthology.orchestra.run.smithi154.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T04:45:48.215 INFO:teuthology.orchestra.run.smithi154.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T04:45:48.215 INFO:teuthology.orchestra.run.smithi154.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-02T04:45:48.216 INFO:teuthology.run_tasks:Running task install... 2024-04-02T04:45:48.225 DEBUG:teuthology.task.install:project ceph 2024-04-02T04:45:48.225 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '6cec793d9073f0594e2364b9b687ecaf22424cca'}} 2024-04-02T04:45:48.225 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': '6cec793d9073f0594e2364b9b687ecaf22424cca'} 2024-04-02T04:45:48.225 INFO:teuthology.task.install:Using flavor: default 2024-04-02T04:45:48.233 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-04-02T04:45:48.233 INFO:teuthology.task.install:extra packages: [] 2024-04-02T04:45:48.234 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '6cec793d9073f0594e2364b9b687ecaf22424cca', 'tag': None, 'wait_for_package': False} 2024-04-02T04:45:48.234 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-02T04:45:48.234 INFO:teuthology.packaging:ref: None 2024-04-02T04:45:48.234 INFO:teuthology.packaging:tag: None 2024-04-02T04:45:48.234 INFO:teuthology.packaging:branch: quincy 2024-04-02T04:45:48.234 INFO:teuthology.packaging:sha1: 6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T04:45:48.234 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-02T04:45:48.236 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '6cec793d9073f0594e2364b9b687ecaf22424cca', 'tag': None, 'wait_for_package': False} 2024-04-02T04:45:48.236 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-02T04:45:48.237 INFO:teuthology.packaging:ref: None 2024-04-02T04:45:48.237 INFO:teuthology.packaging:tag: None 2024-04-02T04:45:48.237 INFO:teuthology.packaging:branch: quincy 2024-04-02T04:45:48.237 INFO:teuthology.packaging:sha1: 6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T04:45:48.237 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-02T04:45:48.472 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/quincy/c924f9914f2809997b363f77c5cbb14929881220/centos/9/flavors/default/ 2024-04-02T04:45:48.473 INFO:teuthology.task.install.rpm:Package version is 17.2.7-803.gc924f991 2024-04-02T04:45:48.575 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/quincy/c924f9914f2809997b363f77c5cbb14929881220/centos/9/flavors/default/ 2024-04-02T04:45:48.576 INFO:teuthology.task.install.rpm:Package version is 17.2.7-803.gc924f991 2024-04-02T04:45:48.645 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/quincy/c924f9914f2809997b363f77c5cbb14929881220/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/c924f9914f2809997b363f77c5cbb14929881220/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/c924f9914f2809997b363f77c5cbb14929881220/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-02T04:45:48.645 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-02T04:45:48.645 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-02T04:45:48.675 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-04-02T04:45:48.675 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-02T04:45:48.675 INFO:teuthology.packaging:ref: None 2024-04-02T04:45:48.675 INFO:teuthology.packaging:tag: None 2024-04-02T04:45:48.675 INFO:teuthology.packaging:branch: quincy 2024-04-02T04:45:48.675 INFO:teuthology.packaging:sha1: 6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T04:45:48.675 DEBUG:teuthology.orchestra.run.smithi152:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-02T04:45:48.683 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/quincy/c924f9914f2809997b363f77c5cbb14929881220/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/c924f9914f2809997b363f77c5cbb14929881220/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/c924f9914f2809997b363f77c5cbb14929881220/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-04-02T04:45:48.683 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-04-02T04:45:48.683 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-04-02T04:45:48.712 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-04-02T04:45:48.712 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-02T04:45:48.712 INFO:teuthology.packaging:ref: None 2024-04-02T04:45:48.712 INFO:teuthology.packaging:tag: None 2024-04-02T04:45:48.712 INFO:teuthology.packaging:branch: quincy 2024-04-02T04:45:48.712 INFO:teuthology.packaging:sha1: 6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T04:45:48.712 DEBUG:teuthology.orchestra.run.smithi154:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-04-02T04:45:48.745 DEBUG:teuthology.orchestra.run.smithi152:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-02T04:45:48.782 DEBUG:teuthology.orchestra.run.smithi154:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-04-02T04:45:48.826 DEBUG:teuthology.orchestra.run.smithi152:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-02T04:45:48.863 DEBUG:teuthology.orchestra.run.smithi154:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-04-02T04:45:48.893 INFO:teuthology.orchestra.run.smithi152.stdout:check_obsoletes = 1 2024-04-02T04:45:48.895 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum clean all 2024-04-02T04:45:48.930 INFO:teuthology.orchestra.run.smithi154.stdout:check_obsoletes = 1 2024-04-02T04:45:48.931 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum clean all 2024-04-02T04:45:49.135 INFO:teuthology.orchestra.run.smithi152.stdout:74 files removed 2024-04-02T04:45:49.162 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-radosgw 2024-04-02T04:45:49.176 INFO:teuthology.orchestra.run.smithi154.stdout:74 files removed 2024-04-02T04:45:49.202 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum -y install ceph-radosgw 2024-04-02T04:45:49.875 INFO:teuthology.orchestra.run.smithi152.stdout:ceph packages for x86_64 189 kB/s | 77 kB 00:00 2024-04-02T04:45:50.048 INFO:teuthology.orchestra.run.smithi154.stdout:ceph packages for x86_64 142 kB/s | 77 kB 00:00 2024-04-02T04:45:50.250 INFO:teuthology.orchestra.run.smithi152.stdout:ceph noarch packages 28 kB/s | 9.2 kB 00:00 2024-04-02T04:45:50.482 INFO:teuthology.orchestra.run.smithi154.stdout:ceph noarch packages 24 kB/s | 9.2 kB 00:00 2024-04-02T04:45:50.593 INFO:teuthology.orchestra.run.smithi152.stdout:ceph source packages 5.9 kB/s | 1.8 kB 00:00 2024-04-02T04:45:50.873 INFO:teuthology.orchestra.run.smithi154.stdout:ceph source packages 5.1 kB/s | 1.8 kB 00:00 2024-04-02T04:45:50.995 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS-9 - AppStream 18 MB/s | 6.7 MB 00:00 2024-04-02T04:45:51.543 INFO:teuthology.orchestra.run.smithi154.stdout:CentOS-9 - AppStream 11 MB/s | 6.7 MB 00:00 2024-04-02T04:45:52.520 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS-9 - BaseOS 4.9 MB/s | 2.0 MB 00:00 2024-04-02T04:45:53.086 INFO:teuthology.orchestra.run.smithi154.stdout:CentOS-9 - BaseOS 4.7 MB/s | 2.0 MB 00:00 2024-04-02T04:45:53.420 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - BaseOS 14 MB/s | 8.0 MB 00:00 2024-04-02T04:45:54.701 INFO:teuthology.orchestra.run.smithi154.stdout:CentOS Stream 9 - BaseOS 6.2 MB/s | 8.0 MB 00:01 2024-04-02T04:45:55.313 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - AppStream 24 MB/s | 19 MB 00:00 2024-04-02T04:45:56.835 INFO:teuthology.orchestra.run.smithi154.stdout:CentOS Stream 9 - AppStream 18 MB/s | 19 MB 00:01 2024-04-02T04:45:59.347 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - CRB 13 MB/s | 6.1 MB 00:00 2024-04-02T04:46:00.690 INFO:teuthology.orchestra.run.smithi152.stdout:CentOS Stream 9 - Extras packages 90 kB/s | 16 kB 00:00 2024-04-02T04:46:01.045 INFO:teuthology.orchestra.run.smithi154.stdout:CentOS Stream 9 - CRB 8.9 MB/s | 6.1 MB 00:00 2024-04-02T04:46:01.691 INFO:teuthology.orchestra.run.smithi152.stdout:Extra Packages for Enterprise Linux 24 MB/s | 21 MB 00:00 2024-04-02T04:46:02.620 INFO:teuthology.orchestra.run.smithi154.stdout:CentOS Stream 9 - Extras packages 66 kB/s | 16 kB 00:00 2024-04-02T04:46:06.641 INFO:teuthology.orchestra.run.smithi152.stdout:lab-extras 30 kB/s | 1.7 kB 00:00 2024-04-02T04:46:09.281 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-02T04:46:09.287 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:46:09.287 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-02T04:46:09.287 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:46:09.287 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-02T04:46:09.287 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-radosgw x86_64 2:17.2.7-803.gc924f991.el9 ceph 14 M 2024-04-02T04:46:09.287 INFO:teuthology.orchestra.run.smithi152.stdout:Upgrading: 2024-04-02T04:46:09.287 INFO:teuthology.orchestra.run.smithi152.stdout: librados2 x86_64 2:17.2.7-803.gc924f991.el9 ceph 3.1 M 2024-04-02T04:46:09.287 INFO:teuthology.orchestra.run.smithi152.stdout: librbd1 x86_64 2:17.2.7-803.gc924f991.el9 ceph 2.9 M 2024-04-02T04:46:09.287 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-02T04:46:09.287 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-base x86_64 2:17.2.7-803.gc924f991.el9 ceph 5.0 M 2024-04-02T04:46:09.287 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-common x86_64 2:17.2.7-803.gc924f991.el9 ceph 22 M 2024-04-02T04:46:09.288 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-selinux x86_64 2:17.2.7-803.gc924f991.el9 ceph 25 k 2024-04-02T04:46:09.288 INFO:teuthology.orchestra.run.smithi152.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-04-02T04:46:09.288 INFO:teuthology.orchestra.run.smithi152.stdout: libcephfs2 x86_64 2:17.2.7-803.gc924f991.el9 ceph 638 k 2024-04-02T04:46:09.288 INFO:teuthology.orchestra.run.smithi152.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-04-02T04:46:09.288 INFO:teuthology.orchestra.run.smithi152.stdout: libpmemobj x86_64 1.12.1-1.el9 CentOS-AppStream 159 k 2024-04-02T04:46:09.288 INFO:teuthology.orchestra.run.smithi152.stdout: librabbitmq x86_64 0.11.0-7.el9 CentOS-AppStream 44 k 2024-04-02T04:46:09.288 INFO:teuthology.orchestra.run.smithi152.stdout: libradosstriper1 x86_64 2:17.2.7-803.gc924f991.el9 ceph 458 k 2024-04-02T04:46:09.288 INFO:teuthology.orchestra.run.smithi152.stdout: librdkafka x86_64 1.6.1-102.el9 CentOS-AppStream 661 k 2024-04-02T04:46:09.288 INFO:teuthology.orchestra.run.smithi152.stdout: librgw2 x86_64 2:17.2.7-803.gc924f991.el9 ceph 4.9 M 2024-04-02T04:46:09.288 INFO:teuthology.orchestra.run.smithi152.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-04-02T04:46:09.288 INFO:teuthology.orchestra.run.smithi152.stdout: lttng-ust x86_64 2.12.0-6.el9 CentOS-AppStream 282 k 2024-04-02T04:46:09.288 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ceph-argparse x86_64 2:17.2.7-803.gc924f991.el9 ceph 45 k 2024-04-02T04:46:09.288 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ceph-common x86_64 2:17.2.7-803.gc924f991.el9 ceph 117 k 2024-04-02T04:46:09.288 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cephfs x86_64 2:17.2.7-803.gc924f991.el9 ceph 156 k 2024-04-02T04:46:09.289 INFO:teuthology.orchestra.run.smithi152.stdout: python3-prettytable noarch 0.7.2-27.el9 CentOS-AppStream 41 k 2024-04-02T04:46:09.289 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rados x86_64 2:17.2.7-803.gc924f991.el9 ceph 320 k 2024-04-02T04:46:09.289 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rbd x86_64 2:17.2.7-803.gc924f991.el9 ceph 292 k 2024-04-02T04:46:09.289 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rgw x86_64 2:17.2.7-803.gc924f991.el9 ceph 99 k 2024-04-02T04:46:09.289 INFO:teuthology.orchestra.run.smithi152.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-04-02T04:46:09.289 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:46:09.289 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-02T04:46:09.289 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:46:09.289 INFO:teuthology.orchestra.run.smithi152.stdout:Install 22 Packages 2024-04-02T04:46:09.289 INFO:teuthology.orchestra.run.smithi152.stdout:Upgrade 2 Packages 2024-04-02T04:46:09.289 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:46:09.290 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 58 M 2024-04-02T04:46:09.290 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-02T04:46:10.518 INFO:teuthology.orchestra.run.smithi152.stdout:(1/24): ceph-base-17.2.7-803.gc924f991.el9.x86_ 5.7 MB/s | 5.0 MB 00:00 2024-04-02T04:46:10.684 INFO:teuthology.orchestra.run.smithi152.stdout:(2/24): ceph-selinux-17.2.7-803.gc924f991.el9.x 151 kB/s | 25 kB 00:00 2024-04-02T04:46:10.834 INFO:teuthology.orchestra.run.smithi152.stdout:(3/24): libcephfs2-17.2.7-803.gc924f991.el9.x86 4.2 MB/s | 638 kB 00:00 2024-04-02T04:46:10.921 INFO:teuthology.orchestra.run.smithi154.stdout:Extra Packages for Enterprise Linux 2.5 MB/s | 21 MB 00:08 2024-04-02T04:46:10.968 INFO:teuthology.orchestra.run.smithi152.stdout:(4/24): libradosstriper1-17.2.7-803.gc924f991.e 3.4 MB/s | 458 kB 00:00 2024-04-02T04:46:11.685 INFO:teuthology.orchestra.run.smithi152.stdout:(5/24): librgw2-17.2.7-803.gc924f991.el9.x86_64 6.8 MB/s | 4.9 MB 00:00 2024-04-02T04:46:11.936 INFO:teuthology.orchestra.run.smithi152.stdout:(6/24): ceph-common-17.2.7-803.gc924f991.el9.x8 9.8 MB/s | 22 MB 00:02 2024-04-02T04:46:11.952 INFO:teuthology.orchestra.run.smithi152.stdout:(7/24): python3-ceph-argparse-17.2.7-803.gc924f 169 kB/s | 45 kB 00:00 2024-04-02T04:46:12.111 INFO:teuthology.orchestra.run.smithi152.stdout:(8/24): ceph-radosgw-17.2.7-803.gc924f991.el9.x 5.7 MB/s | 14 MB 00:02 2024-04-02T04:46:12.136 INFO:teuthology.orchestra.run.smithi152.stdout:(9/24): python3-ceph-common-17.2.7-803.gc924f99 586 kB/s | 117 kB 00:00 2024-04-02T04:46:12.161 INFO:teuthology.orchestra.run.smithi152.stdout:(10/24): python3-cephfs-17.2.7-803.gc924f991.el 748 kB/s | 156 kB 00:00 2024-04-02T04:46:12.187 INFO:teuthology.orchestra.run.smithi152.stdout:(11/24): python3-rados-17.2.7-803.gc924f991.el9 4.2 MB/s | 320 kB 00:00 2024-04-02T04:46:12.212 INFO:teuthology.orchestra.run.smithi152.stdout:(12/24): python3-rbd-17.2.7-803.gc924f991.el9.x 3.8 MB/s | 292 kB 00:00 2024-04-02T04:46:12.254 INFO:teuthology.orchestra.run.smithi152.stdout:(13/24): python3-rgw-17.2.7-803.gc924f991.el9.x 1.1 MB/s | 99 kB 00:00 2024-04-02T04:46:12.329 INFO:teuthology.orchestra.run.smithi152.stdout:(14/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.1 MB/s | 159 kB 00:00 2024-04-02T04:46:12.346 INFO:teuthology.orchestra.run.smithi152.stdout:(15/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 407 kB/s | 44 kB 00:00 2024-04-02T04:46:12.413 INFO:teuthology.orchestra.run.smithi152.stdout:(16/24): librdkafka-1.6.1-102.el9.x86_64.rpm 4.9 MB/s | 661 kB 00:00 2024-04-02T04:46:12.447 INFO:teuthology.orchestra.run.smithi152.stdout:(17/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.4 MB/s | 282 kB 00:00 2024-04-02T04:46:12.464 INFO:teuthology.orchestra.run.smithi152.stdout:(18/24): python3-prettytable-0.7.2-27.el9.noarc 352 kB/s | 41 kB 00:00 2024-04-02T04:46:12.631 INFO:teuthology.orchestra.run.smithi152.stdout:(19/24): liboath-2.6.7-2.el9.x86_64.rpm 264 kB/s | 49 kB 00:00 2024-04-02T04:46:12.656 INFO:teuthology.orchestra.run.smithi152.stdout:(20/24): libunwind-1.6.2-1.el9.x86_64.rpm 351 kB/s | 67 kB 00:00 2024-04-02T04:46:12.773 INFO:teuthology.orchestra.run.smithi152.stdout:(21/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 860 kB/s | 309 kB 00:00 2024-04-02T04:46:12.932 INFO:teuthology.orchestra.run.smithi152.stdout:(22/24): librados2-17.2.7-803.gc924f991.el9.x86 11 MB/s | 3.1 MB 00:00 2024-04-02T04:46:12.973 INFO:teuthology.orchestra.run.smithi152.stdout:(23/24): thrift-0.15.0-2.el9.x86_64.rpm 4.6 MB/s | 1.6 MB 00:00 2024-04-02T04:46:13.299 INFO:teuthology.orchestra.run.smithi152.stdout:(24/24): librbd1-17.2.7-803.gc924f991.el9.x86_6 5.6 MB/s | 2.9 MB 00:00 2024-04-02T04:46:13.303 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:46:13.303 INFO:teuthology.orchestra.run.smithi152.stdout:Total 14 MB/s | 58 MB 00:04 2024-04-02T04:46:13.304 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-02T04:46:13.338 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-02T04:46:13.338 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-02T04:46:13.805 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-02T04:46:13.805 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-02T04:46:15.016 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-02T04:46:15.371 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-04-02T04:46:15.434 INFO:teuthology.orchestra.run.smithi152.stdout: Upgrading : librados2-2:17.2.7-803.gc924f991.el9.x86_64 2/26 2024-04-02T04:46:15.526 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librados2-2:17.2.7-803.gc924f991.el9.x86_64 2/26 2024-04-02T04:46:15.606 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-rados-2:17.2.7-803.gc924f991.el9.x86_64 3/26 2024-04-02T04:46:15.636 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libcephfs2-2:17.2.7-803.gc924f991.el9.x86_64 4/26 2024-04-02T04:46:15.717 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: libcephfs2-2:17.2.7-803.gc924f991.el9.x86_64 4/26 2024-04-02T04:46:15.770 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-04-02T04:46:15.799 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-04-02T04:46:15.921 INFO:teuthology.orchestra.run.smithi154.stdout:lab-extras 29 kB/s | 1.7 kB 00:00 2024-04-02T04:46:16.233 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-04-02T04:46:16.264 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : librgw2-2:17.2.7-803.gc924f991.el9.x86_64 8/26 2024-04-02T04:46:16.335 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librgw2-2:17.2.7-803.gc924f991.el9.x86_64 8/26 2024-04-02T04:46:16.364 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-04-02T04:46:16.407 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-ceph-argparse-2:17.2.7-803.gc924f991.el9.x 10/26 2024-04-02T04:46:16.444 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cephfs-2:17.2.7-803.gc924f991.el9.x86_64 11/26 2024-04-02T04:46:16.506 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-rgw-2:17.2.7-803.gc924f991.el9.x86_64 12/26 2024-04-02T04:46:16.542 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libradosstriper1-2:17.2.7-803.gc924f991.el9.x86_64 13/26 2024-04-02T04:46:16.627 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: libradosstriper1-2:17.2.7-803.gc924f991.el9.x86_64 13/26 2024-04-02T04:46:16.673 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-04-02T04:46:16.743 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-04-02T04:46:16.775 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-04-02T04:46:17.076 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-04-02T04:46:17.105 INFO:teuthology.orchestra.run.smithi152.stdout: Upgrading : librbd1-2:17.2.7-803.gc924f991.el9.x86_64 18/26 2024-04-02T04:46:17.195 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librbd1-2:17.2.7-803.gc924f991.el9.x86_64 18/26 2024-04-02T04:46:17.268 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-rbd-2:17.2.7-803.gc924f991.el9.x86_64 19/26 2024-04-02T04:46:17.299 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-ceph-common-2:17.2.7-803.gc924f991.el9.x86 20/26 2024-04-02T04:46:18.499 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-04-02T04:46:18.537 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:46:18.537 INFO:teuthology.orchestra.run.smithi154.stdout: Package Arch Version Repository Size 2024-04-02T04:46:18.537 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:46:18.537 INFO:teuthology.orchestra.run.smithi154.stdout:Installing: 2024-04-02T04:46:18.537 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-radosgw x86_64 2:17.2.7-803.gc924f991.el9 ceph 14 M 2024-04-02T04:46:18.537 INFO:teuthology.orchestra.run.smithi154.stdout:Upgrading: 2024-04-02T04:46:18.537 INFO:teuthology.orchestra.run.smithi154.stdout: librados2 x86_64 2:17.2.7-803.gc924f991.el9 ceph 3.1 M 2024-04-02T04:46:18.537 INFO:teuthology.orchestra.run.smithi154.stdout: librbd1 x86_64 2:17.2.7-803.gc924f991.el9 ceph 2.9 M 2024-04-02T04:46:18.537 INFO:teuthology.orchestra.run.smithi154.stdout:Installing dependencies: 2024-04-02T04:46:18.537 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-base x86_64 2:17.2.7-803.gc924f991.el9 ceph 5.0 M 2024-04-02T04:46:18.538 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-common x86_64 2:17.2.7-803.gc924f991.el9 ceph 22 M 2024-04-02T04:46:18.538 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-selinux x86_64 2:17.2.7-803.gc924f991.el9 ceph 25 k 2024-04-02T04:46:18.538 INFO:teuthology.orchestra.run.smithi154.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-04-02T04:46:18.538 INFO:teuthology.orchestra.run.smithi154.stdout: libcephfs2 x86_64 2:17.2.7-803.gc924f991.el9 ceph 638 k 2024-04-02T04:46:18.538 INFO:teuthology.orchestra.run.smithi154.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-04-02T04:46:18.538 INFO:teuthology.orchestra.run.smithi154.stdout: libpmemobj x86_64 1.12.1-1.el9 CentOS-AppStream 159 k 2024-04-02T04:46:18.538 INFO:teuthology.orchestra.run.smithi154.stdout: librabbitmq x86_64 0.11.0-7.el9 CentOS-AppStream 44 k 2024-04-02T04:46:18.538 INFO:teuthology.orchestra.run.smithi154.stdout: libradosstriper1 x86_64 2:17.2.7-803.gc924f991.el9 ceph 458 k 2024-04-02T04:46:18.538 INFO:teuthology.orchestra.run.smithi154.stdout: librdkafka x86_64 1.6.1-102.el9 CentOS-AppStream 661 k 2024-04-02T04:46:18.538 INFO:teuthology.orchestra.run.smithi154.stdout: librgw2 x86_64 2:17.2.7-803.gc924f991.el9 ceph 4.9 M 2024-04-02T04:46:18.538 INFO:teuthology.orchestra.run.smithi154.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-04-02T04:46:18.538 INFO:teuthology.orchestra.run.smithi154.stdout: lttng-ust x86_64 2.12.0-6.el9 CentOS-AppStream 282 k 2024-04-02T04:46:18.538 INFO:teuthology.orchestra.run.smithi154.stdout: python3-ceph-argparse x86_64 2:17.2.7-803.gc924f991.el9 ceph 45 k 2024-04-02T04:46:18.538 INFO:teuthology.orchestra.run.smithi154.stdout: python3-ceph-common x86_64 2:17.2.7-803.gc924f991.el9 ceph 117 k 2024-04-02T04:46:18.539 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cephfs x86_64 2:17.2.7-803.gc924f991.el9 ceph 156 k 2024-04-02T04:46:18.539 INFO:teuthology.orchestra.run.smithi154.stdout: python3-prettytable noarch 0.7.2-27.el9 CentOS-AppStream 41 k 2024-04-02T04:46:18.539 INFO:teuthology.orchestra.run.smithi154.stdout: python3-rados x86_64 2:17.2.7-803.gc924f991.el9 ceph 320 k 2024-04-02T04:46:18.539 INFO:teuthology.orchestra.run.smithi154.stdout: python3-rbd x86_64 2:17.2.7-803.gc924f991.el9 ceph 292 k 2024-04-02T04:46:18.539 INFO:teuthology.orchestra.run.smithi154.stdout: python3-rgw x86_64 2:17.2.7-803.gc924f991.el9 ceph 99 k 2024-04-02T04:46:18.539 INFO:teuthology.orchestra.run.smithi154.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-04-02T04:46:18.539 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:46:18.539 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction Summary 2024-04-02T04:46:18.539 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:46:18.539 INFO:teuthology.orchestra.run.smithi154.stdout:Install 22 Packages 2024-04-02T04:46:18.539 INFO:teuthology.orchestra.run.smithi154.stdout:Upgrade 2 Packages 2024-04-02T04:46:18.539 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:46:18.540 INFO:teuthology.orchestra.run.smithi154.stdout:Total download size: 58 M 2024-04-02T04:46:18.540 INFO:teuthology.orchestra.run.smithi154.stdout:Downloading Packages: 2024-04-02T04:46:19.630 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-common-2:17.2.7-803.gc924f991.el9.x86_64 21/26 2024-04-02T04:46:19.671 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-common-2:17.2.7-803.gc924f991.el9.x86_64 21/26 2024-04-02T04:46:19.747 INFO:teuthology.orchestra.run.smithi154.stdout:(1/24): ceph-base-17.2.7-803.gc924f991.el9.x86_ 5.8 MB/s | 5.0 MB 00:00 2024-04-02T04:46:19.806 INFO:teuthology.orchestra.run.smithi154.stdout:(2/24): ceph-selinux-17.2.7-803.gc924f991.el9.x 431 kB/s | 25 kB 00:00 2024-04-02T04:46:19.939 INFO:teuthology.orchestra.run.smithi154.stdout:(3/24): libcephfs2-17.2.7-803.gc924f991.el9.x86 4.7 MB/s | 638 kB 00:00 2024-04-02T04:46:20.031 INFO:teuthology.orchestra.run.smithi154.stdout:(4/24): libradosstriper1-17.2.7-803.gc924f991.e 4.9 MB/s | 458 kB 00:00 2024-04-02T04:46:20.221 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-common-2:17.2.7-803.gc924f991.el9.x86_64 21/26 2024-04-02T04:46:20.258 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-base-2:17.2.7-803.gc924f991.el9.x86_64 22/26 2024-04-02T04:46:20.351 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-base-2:17.2.7-803.gc924f991.el9.x86_64 22/26 2024-04-02T04:46:20.351 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-04-02T04:46:20.351 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-04-02T04:46:20.352 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:46:20.384 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-selinux-2:17.2.7-803.gc924f991.el9.x86_64 23/26 2024-04-02T04:46:20.706 INFO:teuthology.orchestra.run.smithi154.stdout:(5/24): librgw2-17.2.7-803.gc924f991.el9.x86_64 7.3 MB/s | 4.9 MB 00:00 2024-04-02T04:46:20.915 INFO:teuthology.orchestra.run.smithi154.stdout:(6/24): ceph-common-17.2.7-803.gc924f991.el9.x8 11 MB/s | 22 MB 00:02 2024-04-02T04:46:20.940 INFO:teuthology.orchestra.run.smithi154.stdout:(7/24): python3-ceph-argparse-17.2.7-803.gc924f 193 kB/s | 45 kB 00:00 2024-04-02T04:46:20.973 INFO:teuthology.orchestra.run.smithi154.stdout:(8/24): python3-ceph-common-17.2.7-803.gc924f99 2.0 MB/s | 117 kB 00:00 2024-04-02T04:46:20.998 INFO:teuthology.orchestra.run.smithi154.stdout:(9/24): python3-cephfs-17.2.7-803.gc924f991.el9 2.6 MB/s | 156 kB 00:00 2024-04-02T04:46:21.024 INFO:teuthology.orchestra.run.smithi154.stdout:(10/24): python3-rados-17.2.7-803.gc924f991.el9 6.2 MB/s | 320 kB 00:00 2024-04-02T04:46:21.049 INFO:teuthology.orchestra.run.smithi154.stdout:(11/24): python3-rbd-17.2.7-803.gc924f991.el9.x 5.7 MB/s | 292 kB 00:00 2024-04-02T04:46:21.074 INFO:teuthology.orchestra.run.smithi154.stdout:(12/24): python3-rgw-17.2.7-803.gc924f991.el9.x 1.9 MB/s | 99 kB 00:00 2024-04-02T04:46:21.242 INFO:teuthology.orchestra.run.smithi154.stdout:(13/24): ceph-radosgw-17.2.7-803.gc924f991.el9. 6.0 MB/s | 14 MB 00:02 2024-04-02T04:46:21.325 INFO:teuthology.orchestra.run.smithi154.stdout:(14/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 577 kB/s | 159 kB 00:00 2024-04-02T04:46:21.350 INFO:teuthology.orchestra.run.smithi154.stdout:(15/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 160 kB/s | 44 kB 00:00 2024-04-02T04:46:21.393 INFO:teuthology.orchestra.run.smithi154.stdout:(16/24): librdkafka-1.6.1-102.el9.x86_64.rpm 5.2 MB/s | 661 kB 00:00 2024-04-02T04:46:21.418 INFO:teuthology.orchestra.run.smithi154.stdout:(17/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 3.0 MB/s | 282 kB 00:00 2024-04-02T04:46:21.443 INFO:teuthology.orchestra.run.smithi154.stdout:(18/24): python3-prettytable-0.7.2-27.el9.noarc 449 kB/s | 41 kB 00:00 2024-04-02T04:46:21.801 INFO:teuthology.orchestra.run.smithi154.stdout:(19/24): liboath-2.6.7-2.el9.x86_64.rpm 136 kB/s | 49 kB 00:00 2024-04-02T04:46:21.827 INFO:teuthology.orchestra.run.smithi154.stdout:(20/24): libunwind-1.6.2-1.el9.x86_64.rpm 176 kB/s | 67 kB 00:00 2024-04-02T04:46:21.919 INFO:teuthology.orchestra.run.smithi154.stdout:(21/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 588 kB/s | 309 kB 00:00 2024-04-02T04:46:22.086 INFO:teuthology.orchestra.run.smithi154.stdout:(22/24): librados2-17.2.7-803.gc924f991.el9.x86 12 MB/s | 3.1 MB 00:00 2024-04-02T04:46:22.194 INFO:teuthology.orchestra.run.smithi154.stdout:(23/24): thrift-0.15.0-2.el9.x86_64.rpm 4.0 MB/s | 1.6 MB 00:00 2024-04-02T04:46:22.253 INFO:teuthology.orchestra.run.smithi154.stdout:(24/24): librbd1-17.2.7-803.gc924f991.el9.x86_6 8.8 MB/s | 2.9 MB 00:00 2024-04-02T04:46:22.258 INFO:teuthology.orchestra.run.smithi154.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:46:22.261 INFO:teuthology.orchestra.run.smithi154.stdout:Total 16 MB/s | 58 MB 00:03 2024-04-02T04:46:22.262 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction check 2024-04-02T04:46:22.289 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction check succeeded. 2024-04-02T04:46:22.289 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction test 2024-04-02T04:46:22.743 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction test succeeded. 2024-04-02T04:46:22.743 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction 2024-04-02T04:46:23.951 INFO:teuthology.orchestra.run.smithi154.stdout: Preparing : 1/1 2024-04-02T04:46:24.328 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-04-02T04:46:24.363 INFO:teuthology.orchestra.run.smithi154.stdout: Upgrading : librados2-2:17.2.7-803.gc924f991.el9.x86_64 2/26 2024-04-02T04:46:24.455 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: librados2-2:17.2.7-803.gc924f991.el9.x86_64 2/26 2024-04-02T04:46:24.534 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-rados-2:17.2.7-803.gc924f991.el9.x86_64 3/26 2024-04-02T04:46:24.556 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : libcephfs2-2:17.2.7-803.gc924f991.el9.x86_64 4/26 2024-04-02T04:46:24.645 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: libcephfs2-2:17.2.7-803.gc924f991.el9.x86_64 4/26 2024-04-02T04:46:24.708 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-04-02T04:46:24.736 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-04-02T04:46:25.167 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-04-02T04:46:25.285 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : librgw2-2:17.2.7-803.gc924f991.el9.x86_64 8/26 2024-04-02T04:46:25.356 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: librgw2-2:17.2.7-803.gc924f991.el9.x86_64 8/26 2024-04-02T04:46:25.385 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-04-02T04:46:25.427 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-ceph-argparse-2:17.2.7-803.gc924f991.el9.x 10/26 2024-04-02T04:46:25.473 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-cephfs-2:17.2.7-803.gc924f991.el9.x86_64 11/26 2024-04-02T04:46:25.535 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-rgw-2:17.2.7-803.gc924f991.el9.x86_64 12/26 2024-04-02T04:46:25.563 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : libradosstriper1-2:17.2.7-803.gc924f991.el9.x86_64 13/26 2024-04-02T04:46:25.644 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: libradosstriper1-2:17.2.7-803.gc924f991.el9.x86_64 13/26 2024-04-02T04:46:25.686 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-04-02T04:46:25.730 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-04-02T04:46:25.763 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-04-02T04:46:26.059 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-04-02T04:46:26.084 INFO:teuthology.orchestra.run.smithi154.stdout: Upgrading : librbd1-2:17.2.7-803.gc924f991.el9.x86_64 18/26 2024-04-02T04:46:26.173 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: librbd1-2:17.2.7-803.gc924f991.el9.x86_64 18/26 2024-04-02T04:46:26.249 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-rbd-2:17.2.7-803.gc924f991.el9.x86_64 19/26 2024-04-02T04:46:26.278 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-ceph-common-2:17.2.7-803.gc924f991.el9.x86 20/26 2024-04-02T04:46:28.710 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-common-2:17.2.7-803.gc924f991.el9.x86_64 21/26 2024-04-02T04:46:28.750 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-common-2:17.2.7-803.gc924f991.el9.x86_64 21/26 2024-04-02T04:46:29.288 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-common-2:17.2.7-803.gc924f991.el9.x86_64 21/26 2024-04-02T04:46:29.328 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-base-2:17.2.7-803.gc924f991.el9.x86_64 22/26 2024-04-02T04:46:29.410 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-base-2:17.2.7-803.gc924f991.el9.x86_64 22/26 2024-04-02T04:46:29.410 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-04-02T04:46:29.411 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-04-02T04:46:29.411 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:46:29.480 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-selinux-2:17.2.7-803.gc924f991.el9.x86_64 23/26 2024-04-02T04:46:30.121 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-selinux-2:17.2.7-803.gc924f991.el9.x86_64 23/26 2024-04-02T04:46:30.122 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /sys 2024-04-02T04:46:30.122 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /proc 2024-04-02T04:46:30.122 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /mnt 2024-04-02T04:46:30.122 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /var/tmp 2024-04-02T04:46:30.122 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /home 2024-04-02T04:46:30.122 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /root 2024-04-02T04:46:30.122 INFO:teuthology.orchestra.run.smithi152.stdout:skipping the directory /tmp 2024-04-02T04:46:30.122 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:46:31.390 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-radosgw-2:17.2.7-803.gc924f991.el9.x86_64 24/26 2024-04-02T04:46:31.465 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-803.gc924f991.el9.x86_64 24/26 2024-04-02T04:46:31.466 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-02T04:46:31.466 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-04-02T04:46:31.466 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-02T04:46:31.466 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-02T04:46:31.466 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:46:31.467 INFO:teuthology.orchestra.run.smithi152.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-02T04:46:31.544 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-02T04:46:31.544 INFO:teuthology.orchestra.run.smithi152.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-02T04:46:32.757 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-02T04:46:32.757 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-base-2:17.2.7-803.gc924f991.el9.x86_64 1/26 2024-04-02T04:46:32.757 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-common-2:17.2.7-803.gc924f991.el9.x86_64 2/26 2024-04-02T04:46:32.757 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-radosgw-2:17.2.7-803.gc924f991.el9.x86_64 3/26 2024-04-02T04:46:32.757 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-selinux-2:17.2.7-803.gc924f991.el9.x86_64 4/26 2024-04-02T04:46:32.757 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libcephfs2-2:17.2.7-803.gc924f991.el9.x86_64 5/26 2024-04-02T04:46:32.757 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libradosstriper1-2:17.2.7-803.gc924f991.el9.x86_64 6/26 2024-04-02T04:46:32.757 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librgw2-2:17.2.7-803.gc924f991.el9.x86_64 7/26 2024-04-02T04:46:32.757 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-ceph-argparse-2:17.2.7-803.gc924f991.el9.x 8/26 2024-04-02T04:46:32.757 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-ceph-common-2:17.2.7-803.gc924f991.el9.x86 9/26 2024-04-02T04:46:32.757 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cephfs-2:17.2.7-803.gc924f991.el9.x86_64 10/26 2024-04-02T04:46:32.758 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-rados-2:17.2.7-803.gc924f991.el9.x86_64 11/26 2024-04-02T04:46:32.758 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-rbd-2:17.2.7-803.gc924f991.el9.x86_64 12/26 2024-04-02T04:46:32.758 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-rgw-2:17.2.7-803.gc924f991.el9.x86_64 13/26 2024-04-02T04:46:32.758 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-04-02T04:46:32.758 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-04-02T04:46:32.758 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-04-02T04:46:32.758 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-04-02T04:46:32.758 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-04-02T04:46:32.758 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-04-02T04:46:32.758 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-04-02T04:46:32.758 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-04-02T04:46:32.758 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-04-02T04:46:32.758 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librados2-2:17.2.7-803.gc924f991.el9.x86_64 23/26 2024-04-02T04:46:32.758 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-04-02T04:46:32.759 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librbd1-2:17.2.7-803.gc924f991.el9.x86_64 25/26 2024-04-02T04:46:33.695 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-04-02T04:46:33.696 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:46:33.696 INFO:teuthology.orchestra.run.smithi152.stdout:Upgraded: 2024-04-02T04:46:33.696 INFO:teuthology.orchestra.run.smithi152.stdout: librados2-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:33.696 INFO:teuthology.orchestra.run.smithi152.stdout: librbd1-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:33.696 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-02T04:46:33.696 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-base-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:33.696 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-common-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:33.696 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-radosgw-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:33.696 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-selinux-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:33.696 INFO:teuthology.orchestra.run.smithi152.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-04-02T04:46:33.696 INFO:teuthology.orchestra.run.smithi152.stdout: libcephfs2-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:33.696 INFO:teuthology.orchestra.run.smithi152.stdout: liboath-2.6.7-2.el9.x86_64 2024-04-02T04:46:33.696 INFO:teuthology.orchestra.run.smithi152.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-04-02T04:46:33.696 INFO:teuthology.orchestra.run.smithi152.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-04-02T04:46:33.697 INFO:teuthology.orchestra.run.smithi152.stdout: libradosstriper1-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:33.697 INFO:teuthology.orchestra.run.smithi152.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-04-02T04:46:33.697 INFO:teuthology.orchestra.run.smithi152.stdout: librgw2-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:33.697 INFO:teuthology.orchestra.run.smithi152.stdout: libunwind-1.6.2-1.el9.x86_64 2024-04-02T04:46:33.697 INFO:teuthology.orchestra.run.smithi152.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-04-02T04:46:33.697 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ceph-argparse-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:33.697 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ceph-common-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:33.697 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cephfs-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:33.697 INFO:teuthology.orchestra.run.smithi152.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-04-02T04:46:33.697 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rados-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:33.697 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rbd-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:33.697 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rgw-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:33.697 INFO:teuthology.orchestra.run.smithi152.stdout: thrift-0.15.0-2.el9.x86_64 2024-04-02T04:46:33.697 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:46:33.698 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-02T04:46:34.031 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-test 2024-04-02T04:46:34.593 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:28 ago on Tue 02 Apr 2024 04:46:06 AM UTC. 2024-04-02T04:46:34.749 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-02T04:46:34.750 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:46:34.750 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-02T04:46:34.750 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:46:34.751 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-02T04:46:34.751 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-test x86_64 2:17.2.7-803.gc924f991.el9 ceph 48 M 2024-04-02T04:46:34.751 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-02T04:46:34.751 INFO:teuthology.orchestra.run.smithi152.stdout: jq x86_64 1.6-16.el9 baseos 187 k 2024-04-02T04:46:34.751 INFO:teuthology.orchestra.run.smithi152.stdout: libcephsqlite x86_64 2:17.2.7-803.gc924f991.el9 ceph 163 k 2024-04-02T04:46:34.751 INFO:teuthology.orchestra.run.smithi152.stdout: oniguruma x86_64 6.9.6-1.el9.5 CentOS-AppStream 217 k 2024-04-02T04:46:34.751 INFO:teuthology.orchestra.run.smithi152.stdout: socat x86_64 1.7.4.1-5.el9 CentOS-AppStream 300 k 2024-04-02T04:46:34.751 INFO:teuthology.orchestra.run.smithi152.stdout: xmlstarlet x86_64 1.6.1-20.el9 CentOS-AppStream 63 k 2024-04-02T04:46:34.751 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:46:34.751 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-02T04:46:34.751 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:46:34.751 INFO:teuthology.orchestra.run.smithi152.stdout:Install 6 Packages 2024-04-02T04:46:34.751 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:46:34.752 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 49 M 2024-04-02T04:46:34.752 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 193 M 2024-04-02T04:46:34.752 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-02T04:46:34.954 INFO:teuthology.orchestra.run.smithi152.stdout:(1/6): oniguruma-6.9.6-1.el9.5.x86_64.rpm 1.3 MB/s | 217 kB 00:00 2024-04-02T04:46:35.004 INFO:teuthology.orchestra.run.smithi152.stdout:(2/6): socat-1.7.4.1-5.el9.x86_64.rpm 5.9 MB/s | 300 kB 00:00 2024-04-02T04:46:35.038 INFO:teuthology.orchestra.run.smithi152.stdout:(3/6): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.8 MB/s | 63 kB 00:00 2024-04-02T04:46:35.204 INFO:teuthology.orchestra.run.smithi152.stdout:(4/6): jq-1.6-16.el9.x86_64.rpm 1.1 MB/s | 187 kB 00:00 2024-04-02T04:46:35.371 INFO:teuthology.orchestra.run.smithi152.stdout:(5/6): libcephsqlite-17.2.7-803.gc924f991.el9.x 279 kB/s | 163 kB 00:00 2024-04-02T04:46:38.530 INFO:teuthology.orchestra.run.smithi152.stdout:(6/6): ceph-test-17.2.7-803.gc924f991.el9.x86_6 13 MB/s | 48 MB 00:03 2024-04-02T04:46:38.532 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:46:38.533 INFO:teuthology.orchestra.run.smithi152.stdout:Total 13 MB/s | 49 MB 00:03 2024-04-02T04:46:38.547 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-02T04:46:38.568 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-02T04:46:38.568 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-02T04:46:38.935 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-selinux-2:17.2.7-803.gc924f991.el9.x86_64 23/26 2024-04-02T04:46:38.935 INFO:teuthology.orchestra.run.smithi154.stdout:skipping the directory /sys 2024-04-02T04:46:38.935 INFO:teuthology.orchestra.run.smithi154.stdout:skipping the directory /proc 2024-04-02T04:46:38.935 INFO:teuthology.orchestra.run.smithi154.stdout:skipping the directory /mnt 2024-04-02T04:46:38.935 INFO:teuthology.orchestra.run.smithi154.stdout:skipping the directory /var/tmp 2024-04-02T04:46:38.935 INFO:teuthology.orchestra.run.smithi154.stdout:skipping the directory /home 2024-04-02T04:46:38.935 INFO:teuthology.orchestra.run.smithi154.stdout:skipping the directory /root 2024-04-02T04:46:38.936 INFO:teuthology.orchestra.run.smithi154.stdout:skipping the directory /tmp 2024-04-02T04:46:38.936 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:46:38.960 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-02T04:46:38.961 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-02T04:46:39.538 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-02T04:46:39.631 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/6 2024-04-02T04:46:39.668 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/6 2024-04-02T04:46:39.729 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-04-02T04:46:39.753 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : jq-1.6-16.el9.x86_64 4/6 2024-04-02T04:46:39.783 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libcephsqlite-2:17.2.7-803.gc924f991.el9.x86_64 5/6 2024-04-02T04:46:40.209 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-radosgw-2:17.2.7-803.gc924f991.el9.x86_64 24/26 2024-04-02T04:46:40.270 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-803.gc924f991.el9.x86_64 24/26 2024-04-02T04:46:40.270 INFO:teuthology.orchestra.run.smithi154.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-02T04:46:40.271 INFO:teuthology.orchestra.run.smithi154.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-04-02T04:46:40.271 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-02T04:46:40.271 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-04-02T04:46:40.271 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:46:40.272 INFO:teuthology.orchestra.run.smithi154.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-02T04:46:40.370 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-04-02T04:46:40.370 INFO:teuthology.orchestra.run.smithi154.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-02T04:46:41.594 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-04-02T04:46:41.594 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-base-2:17.2.7-803.gc924f991.el9.x86_64 1/26 2024-04-02T04:46:41.594 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-common-2:17.2.7-803.gc924f991.el9.x86_64 2/26 2024-04-02T04:46:41.594 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-radosgw-2:17.2.7-803.gc924f991.el9.x86_64 3/26 2024-04-02T04:46:41.594 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-selinux-2:17.2.7-803.gc924f991.el9.x86_64 4/26 2024-04-02T04:46:41.594 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : libcephfs2-2:17.2.7-803.gc924f991.el9.x86_64 5/26 2024-04-02T04:46:41.594 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : libradosstriper1-2:17.2.7-803.gc924f991.el9.x86_64 6/26 2024-04-02T04:46:41.594 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : librgw2-2:17.2.7-803.gc924f991.el9.x86_64 7/26 2024-04-02T04:46:41.594 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-ceph-argparse-2:17.2.7-803.gc924f991.el9.x 8/26 2024-04-02T04:46:41.594 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-ceph-common-2:17.2.7-803.gc924f991.el9.x86 9/26 2024-04-02T04:46:41.594 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-cephfs-2:17.2.7-803.gc924f991.el9.x86_64 10/26 2024-04-02T04:46:41.594 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-rados-2:17.2.7-803.gc924f991.el9.x86_64 11/26 2024-04-02T04:46:41.595 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-rbd-2:17.2.7-803.gc924f991.el9.x86_64 12/26 2024-04-02T04:46:41.595 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-rgw-2:17.2.7-803.gc924f991.el9.x86_64 13/26 2024-04-02T04:46:41.595 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-04-02T04:46:41.595 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-04-02T04:46:41.595 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-04-02T04:46:41.595 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-04-02T04:46:41.595 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-04-02T04:46:41.595 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-04-02T04:46:41.595 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-04-02T04:46:41.595 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-04-02T04:46:41.595 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-04-02T04:46:41.595 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : librados2-2:17.2.7-803.gc924f991.el9.x86_64 23/26 2024-04-02T04:46:41.595 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-04-02T04:46:41.595 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : librbd1-2:17.2.7-803.gc924f991.el9.x86_64 25/26 2024-04-02T04:46:42.527 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-04-02T04:46:42.528 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:46:42.528 INFO:teuthology.orchestra.run.smithi154.stdout:Upgraded: 2024-04-02T04:46:42.528 INFO:teuthology.orchestra.run.smithi154.stdout: librados2-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:42.528 INFO:teuthology.orchestra.run.smithi154.stdout: librbd1-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:42.528 INFO:teuthology.orchestra.run.smithi154.stdout:Installed: 2024-04-02T04:46:42.528 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-base-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:42.528 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-common-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:42.528 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-radosgw-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:42.528 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-selinux-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:42.528 INFO:teuthology.orchestra.run.smithi154.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-04-02T04:46:42.528 INFO:teuthology.orchestra.run.smithi154.stdout: libcephfs2-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:42.528 INFO:teuthology.orchestra.run.smithi154.stdout: liboath-2.6.7-2.el9.x86_64 2024-04-02T04:46:42.528 INFO:teuthology.orchestra.run.smithi154.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-04-02T04:46:42.528 INFO:teuthology.orchestra.run.smithi154.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-04-02T04:46:42.529 INFO:teuthology.orchestra.run.smithi154.stdout: libradosstriper1-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:42.529 INFO:teuthology.orchestra.run.smithi154.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-04-02T04:46:42.529 INFO:teuthology.orchestra.run.smithi154.stdout: librgw2-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:42.529 INFO:teuthology.orchestra.run.smithi154.stdout: libunwind-1.6.2-1.el9.x86_64 2024-04-02T04:46:42.529 INFO:teuthology.orchestra.run.smithi154.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-04-02T04:46:42.529 INFO:teuthology.orchestra.run.smithi154.stdout: python3-ceph-argparse-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:42.529 INFO:teuthology.orchestra.run.smithi154.stdout: python3-ceph-common-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:42.529 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cephfs-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:42.529 INFO:teuthology.orchestra.run.smithi154.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-04-02T04:46:42.529 INFO:teuthology.orchestra.run.smithi154.stdout: python3-rados-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:42.529 INFO:teuthology.orchestra.run.smithi154.stdout: python3-rbd-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:42.529 INFO:teuthology.orchestra.run.smithi154.stdout: python3-rgw-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:42.529 INFO:teuthology.orchestra.run.smithi154.stdout: thrift-0.15.0-2.el9.x86_64 2024-04-02T04:46:42.529 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:46:42.529 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-04-02T04:46:42.882 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum -y install ceph-test 2024-04-02T04:46:43.438 INFO:teuthology.orchestra.run.smithi154.stdout:Last metadata expiration check: 0:00:28 ago on Tue 02 Apr 2024 04:46:15 AM UTC. 2024-04-02T04:46:43.595 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-04-02T04:46:43.596 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:46:43.596 INFO:teuthology.orchestra.run.smithi154.stdout: Package Arch Version Repository Size 2024-04-02T04:46:43.596 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:46:43.596 INFO:teuthology.orchestra.run.smithi154.stdout:Installing: 2024-04-02T04:46:43.596 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-test x86_64 2:17.2.7-803.gc924f991.el9 ceph 48 M 2024-04-02T04:46:43.596 INFO:teuthology.orchestra.run.smithi154.stdout:Installing dependencies: 2024-04-02T04:46:43.596 INFO:teuthology.orchestra.run.smithi154.stdout: jq x86_64 1.6-16.el9 baseos 187 k 2024-04-02T04:46:43.596 INFO:teuthology.orchestra.run.smithi154.stdout: libcephsqlite x86_64 2:17.2.7-803.gc924f991.el9 ceph 163 k 2024-04-02T04:46:43.596 INFO:teuthology.orchestra.run.smithi154.stdout: oniguruma x86_64 6.9.6-1.el9.5 CentOS-AppStream 217 k 2024-04-02T04:46:43.596 INFO:teuthology.orchestra.run.smithi154.stdout: socat x86_64 1.7.4.1-5.el9 CentOS-AppStream 300 k 2024-04-02T04:46:43.596 INFO:teuthology.orchestra.run.smithi154.stdout: xmlstarlet x86_64 1.6.1-20.el9 CentOS-AppStream 63 k 2024-04-02T04:46:43.597 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:46:43.597 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction Summary 2024-04-02T04:46:43.597 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:46:43.597 INFO:teuthology.orchestra.run.smithi154.stdout:Install 6 Packages 2024-04-02T04:46:43.597 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:46:43.597 INFO:teuthology.orchestra.run.smithi154.stdout:Total download size: 49 M 2024-04-02T04:46:43.597 INFO:teuthology.orchestra.run.smithi154.stdout:Installed size: 193 M 2024-04-02T04:46:43.598 INFO:teuthology.orchestra.run.smithi154.stdout:Downloading Packages: 2024-04-02T04:46:43.852 INFO:teuthology.orchestra.run.smithi154.stdout:(1/6): oniguruma-6.9.6-1.el9.5.x86_64.rpm 1.5 MB/s | 217 kB 00:00 2024-04-02T04:46:43.902 INFO:teuthology.orchestra.run.smithi154.stdout:(2/6): socat-1.7.4.1-5.el9.x86_64.rpm 6.0 MB/s | 300 kB 00:00 2024-04-02T04:46:43.936 INFO:teuthology.orchestra.run.smithi154.stdout:(3/6): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.8 MB/s | 63 kB 00:00 2024-04-02T04:46:43.961 INFO:teuthology.orchestra.run.smithi154.stdout:(4/6): libcephsqlite-17.2.7-803.gc924f991.el9.x 638 kB/s | 163 kB 00:00 2024-04-02T04:46:44.151 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: libcephsqlite-2:17.2.7-803.gc924f991.el9.x86_64 5/6 2024-04-02T04:46:44.153 INFO:teuthology.orchestra.run.smithi154.stdout:(5/6): jq-1.6-16.el9.x86_64.rpm 863 kB/s | 187 kB 00:00 2024-04-02T04:46:44.231 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-test-2:17.2.7-803.gc924f991.el9.x86_64 6/6 2024-04-02T04:46:46.262 INFO:teuthology.orchestra.run.smithi154.stdout:(6/6): ceph-test-17.2.7-803.gc924f991.el9.x86_6 19 MB/s | 48 MB 00:02 2024-04-02T04:46:46.265 INFO:teuthology.orchestra.run.smithi154.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:46:46.265 INFO:teuthology.orchestra.run.smithi154.stdout:Total 18 MB/s | 49 MB 00:02 2024-04-02T04:46:46.280 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction check 2024-04-02T04:46:46.301 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction check succeeded. 2024-04-02T04:46:46.301 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction test 2024-04-02T04:46:46.404 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-test-2:17.2.7-803.gc924f991.el9.x86_64 6/6 2024-04-02T04:46:46.404 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-test-2:17.2.7-803.gc924f991.el9.x86_64 1/6 2024-04-02T04:46:46.405 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libcephsqlite-2:17.2.7-803.gc924f991.el9.x86_64 2/6 2024-04-02T04:46:46.405 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-04-02T04:46:46.405 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 4/6 2024-04-02T04:46:46.405 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/6 2024-04-02T04:46:46.690 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction test succeeded. 2024-04-02T04:46:46.690 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction 2024-04-02T04:46:46.780 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : jq-1.6-16.el9.x86_64 6/6 2024-04-02T04:46:46.780 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:46:46.780 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-02T04:46:46.780 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-test-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:46.780 INFO:teuthology.orchestra.run.smithi152.stdout: jq-1.6-16.el9.x86_64 2024-04-02T04:46:46.781 INFO:teuthology.orchestra.run.smithi152.stdout: libcephsqlite-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:46.781 INFO:teuthology.orchestra.run.smithi152.stdout: oniguruma-6.9.6-1.el9.5.x86_64 2024-04-02T04:46:46.781 INFO:teuthology.orchestra.run.smithi152.stdout: socat-1.7.4.1-5.el9.x86_64 2024-04-02T04:46:46.781 INFO:teuthology.orchestra.run.smithi152.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-04-02T04:46:46.781 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:46:46.781 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-02T04:46:47.088 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph 2024-04-02T04:46:47.275 INFO:teuthology.orchestra.run.smithi154.stdout: Preparing : 1/1 2024-04-02T04:46:47.357 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/6 2024-04-02T04:46:47.405 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/6 2024-04-02T04:46:47.446 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-04-02T04:46:47.489 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : jq-1.6-16.el9.x86_64 4/6 2024-04-02T04:46:47.515 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : libcephsqlite-2:17.2.7-803.gc924f991.el9.x86_64 5/6 2024-04-02T04:46:47.663 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:41 ago on Tue 02 Apr 2024 04:46:06 AM UTC. 2024-04-02T04:46:47.826 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-02T04:46:47.828 INFO:teuthology.orchestra.run.smithi152.stdout:======================================================================================== 2024-04-02T04:46:47.828 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-02T04:46:47.828 INFO:teuthology.orchestra.run.smithi152.stdout:======================================================================================== 2024-04-02T04:46:47.828 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-02T04:46:47.828 INFO:teuthology.orchestra.run.smithi152.stdout: ceph x86_64 2:17.2.7-803.gc924f991.el9 ceph 6.5 k 2024-04-02T04:46:47.828 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-02T04:46:47.828 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mds x86_64 2:17.2.7-803.gc924f991.el9 ceph 2.1 M 2024-04-02T04:46:47.829 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr x86_64 2:17.2.7-803.gc924f991.el9 ceph 1.4 M 2024-04-02T04:46:47.829 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-modules-core noarch 2:17.2.7-803.gc924f991.el9 ceph-noarch 240 k 2024-04-02T04:46:47.829 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mon x86_64 2:17.2.7-803.gc924f991.el9 ceph 3.9 M 2024-04-02T04:46:47.829 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-osd x86_64 2:17.2.7-803.gc924f991.el9 ceph 15 M 2024-04-02T04:46:47.829 INFO:teuthology.orchestra.run.smithi152.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-04-02T04:46:47.829 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cffi x86_64 1.14.5-5.el9 CentOS-BaseOS 241 k 2024-04-02T04:46:47.829 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-04-02T04:46:47.829 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-04-02T04:46:47.829 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cryptography x86_64 36.0.1-4.el9 CentOS-BaseOS 1.1 M 2024-04-02T04:46:47.829 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-04-02T04:46:47.829 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-04-02T04:46:47.829 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-04-02T04:46:47.829 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-04-02T04:46:47.829 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-04-02T04:46:47.830 INFO:teuthology.orchestra.run.smithi152.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-04-02T04:46:47.830 INFO:teuthology.orchestra.run.smithi152.stdout: python3-mako noarch 1.1.4-6.el9 CentOS-AppStream 144 k 2024-04-02T04:46:47.830 INFO:teuthology.orchestra.run.smithi152.stdout: python3-markupsafe x86_64 1.1.1-12.el9 CentOS-AppStream 32 k 2024-04-02T04:46:47.830 INFO:teuthology.orchestra.run.smithi152.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-04-02T04:46:47.830 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-04-02T04:46:47.830 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ply noarch 3.11-14.el9 CentOS-BaseOS 103 k 2024-04-02T04:46:47.830 INFO:teuthology.orchestra.run.smithi152.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-04-02T04:46:47.830 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-04-02T04:46:47.830 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pycparser noarch 2.20-6.el9 CentOS-BaseOS 124 k 2024-04-02T04:46:47.830 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pytz noarch 2021.1-5.el9 CentOS-AppStream 47 k 2024-04-02T04:46:47.830 INFO:teuthology.orchestra.run.smithi152.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-04-02T04:46:47.830 INFO:teuthology.orchestra.run.smithi152.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-04-02T04:46:47.830 INFO:teuthology.orchestra.run.smithi152.stdout: python3-toml noarch 0.10.2-6.el9 CentOS-AppStream 39 k 2024-04-02T04:46:47.830 INFO:teuthology.orchestra.run.smithi152.stdout: python3-urllib3 noarch 1.26.5-5.el9 CentOS-BaseOS 187 k 2024-04-02T04:46:47.831 INFO:teuthology.orchestra.run.smithi152.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-04-02T04:46:47.831 INFO:teuthology.orchestra.run.smithi152.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-04-02T04:46:47.831 INFO:teuthology.orchestra.run.smithi152.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-04-02T04:46:47.831 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:46:47.831 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-02T04:46:47.831 INFO:teuthology.orchestra.run.smithi152.stdout:======================================================================================== 2024-04-02T04:46:47.831 INFO:teuthology.orchestra.run.smithi152.stdout:Install 33 Packages 2024-04-02T04:46:47.831 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:46:47.832 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 27 M 2024-04-02T04:46:47.832 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 96 M 2024-04-02T04:46:47.832 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-02T04:46:48.430 INFO:teuthology.orchestra.run.smithi152.stdout:(1/33): ceph-17.2.7-803.gc924f991.el9.x86_64.rp 41 kB/s | 6.5 kB 00:00 2024-04-02T04:46:48.664 INFO:teuthology.orchestra.run.smithi152.stdout:(2/33): ceph-mds-17.2.7-803.gc924f991.el9.x86_6 5.3 MB/s | 2.1 MB 00:00 2024-04-02T04:46:48.956 INFO:teuthology.orchestra.run.smithi152.stdout:(3/33): ceph-mon-17.2.7-803.gc924f991.el9.x86_6 7.5 MB/s | 3.9 MB 00:00 2024-04-02T04:46:49.081 INFO:teuthology.orchestra.run.smithi152.stdout:(4/33): ceph-mgr-modules-core-17.2.7-803.gc924f 1.9 MB/s | 240 kB 00:00 2024-04-02T04:46:49.115 INFO:teuthology.orchestra.run.smithi152.stdout:(5/33): ceph-mgr-17.2.7-803.gc924f991.el9.x86_6 1.7 MB/s | 1.4 MB 00:00 2024-04-02T04:46:49.231 INFO:teuthology.orchestra.run.smithi152.stdout:(6/33): python3-markupsafe-1.1.1-12.el9.x86_64. 270 kB/s | 32 kB 00:00 2024-04-02T04:46:49.257 INFO:teuthology.orchestra.run.smithi152.stdout:(7/33): python3-mako-1.1.4-6.el9.noarch.rpm 822 kB/s | 144 kB 00:00 2024-04-02T04:46:49.282 INFO:teuthology.orchestra.run.smithi152.stdout:(8/33): python3-pytz-2021.1-5.el9.noarch.rpm 948 kB/s | 47 kB 00:00 2024-04-02T04:46:49.307 INFO:teuthology.orchestra.run.smithi152.stdout:(9/33): python3-toml-0.10.2-6.el9.noarch.rpm 771 kB/s | 39 kB 00:00 2024-04-02T04:46:49.341 INFO:teuthology.orchestra.run.smithi152.stdout:(10/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 4.0 MB/s | 241 kB 00:00 2024-04-02T04:46:49.500 INFO:teuthology.orchestra.run.smithi152.stdout:(11/33): ceph-osd-17.2.7-803.gc924f991.el9.x86_ 18 MB/s | 15 MB 00:00 2024-04-02T04:46:49.533 INFO:teuthology.orchestra.run.smithi152.stdout:(12/33): python3-cryptography-36.0.1-4.el9.x86_ 5.1 MB/s | 1.1 MB 00:00 2024-04-02T04:46:49.558 INFO:teuthology.orchestra.run.smithi152.stdout:(13/33): python3-ply-3.11-14.el9.noarch.rpm 475 kB/s | 103 kB 00:00 2024-04-02T04:46:49.583 INFO:teuthology.orchestra.run.smithi152.stdout:(14/33): python3-pycparser-2.20-6.el9.noarch.rp 1.5 MB/s | 124 kB 00:00 2024-04-02T04:46:49.609 INFO:teuthology.orchestra.run.smithi152.stdout:(15/33): python3-urllib3-1.26.5-5.el9.noarch.rp 3.7 MB/s | 187 kB 00:00 2024-04-02T04:46:49.801 INFO:teuthology.orchestra.run.smithi152.stdout:(16/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 200 kB/s | 43 kB 00:00 2024-04-02T04:46:49.884 INFO:teuthology.orchestra.run.smithi152.stdout:(17/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 624 kB/s | 172 kB 00:00 2024-04-02T04:46:49.926 INFO:teuthology.orchestra.run.smithi152.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 258 kB/s | 11 kB 00:00 2024-04-02T04:46:49.968 INFO:teuthology.orchestra.run.smithi152.stdout:(19/33): python3-jaraco-classes-3.2.1-5.el9.noa 422 kB/s | 18 kB 00:00 2024-04-02T04:46:50.002 INFO:teuthology.orchestra.run.smithi152.stdout:(20/33): python3-cherrypy-18.6.1-2.el9.noarch.r 1.7 MB/s | 358 kB 00:00 2024-04-02T04:46:50.027 INFO:teuthology.orchestra.run.smithi152.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 396 kB/s | 23 kB 00:00 2024-04-02T04:46:50.052 INFO:teuthology.orchestra.run.smithi152.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 389 kB/s | 19 kB 00:00 2024-04-02T04:46:50.077 INFO:teuthology.orchestra.run.smithi152.stdout:(23/33): python3-requests-2.25.1-8.el9.noarch.r 241 kB/s | 125 kB 00:00 2024-04-02T04:46:50.102 INFO:teuthology.orchestra.run.smithi152.stdout:(24/33): python3-jaraco-text-3.2.0-6.el9.noarch 261 kB/s | 20 kB 00:00 2024-04-02T04:46:50.128 INFO:teuthology.orchestra.run.smithi152.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 614 kB/s | 46 kB 00:00 2024-04-02T04:46:50.178 INFO:teuthology.orchestra.run.smithi152.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 1.0 MB/s | 79 kB 00:00 2024-04-02T04:46:50.203 INFO:teuthology.orchestra.run.smithi152.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 326 kB/s | 16 kB 00:00 2024-04-02T04:46:50.237 INFO:teuthology.orchestra.run.smithi152.stdout:(28/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.5 MB/s | 90 kB 00:00 2024-04-02T04:46:50.262 INFO:teuthology.orchestra.run.smithi152.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 610 kB/s | 36 kB 00:00 2024-04-02T04:46:50.346 INFO:teuthology.orchestra.run.smithi152.stdout:(30/33): python3-webob-1.8.7-6.el9.noarch.rpm 2.1 MB/s | 230 kB 00:00 2024-04-02T04:46:50.379 INFO:teuthology.orchestra.run.smithi152.stdout:(31/33): python3-pecan-1.4.2-3.el9.noarch.rpm 985 kB/s | 272 kB 00:00 2024-04-02T04:46:50.405 INFO:teuthology.orchestra.run.smithi152.stdout:(32/33): python3-zc-lockfile-2.0-10.el9.noarch. 343 kB/s | 20 kB 00:00 2024-04-02T04:46:50.472 INFO:teuthology.orchestra.run.smithi152.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 2.0 MB/s | 427 kB 00:00 2024-04-02T04:46:50.478 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:46:50.478 INFO:teuthology.orchestra.run.smithi152.stdout:Total 10 MB/s | 27 MB 00:02 2024-04-02T04:46:50.492 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-02T04:46:50.530 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-02T04:46:50.530 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-02T04:46:50.873 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-02T04:46:50.873 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-02T04:46:51.894 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: libcephsqlite-2:17.2.7-803.gc924f991.el9.x86_64 5/6 2024-04-02T04:46:51.966 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-test-2:17.2.7-803.gc924f991.el9.x86_64 6/6 2024-04-02T04:46:52.105 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-02T04:46:52.174 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-04-02T04:46:52.206 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-04-02T04:46:52.239 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-04-02T04:46:52.332 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-04-02T04:46:52.409 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-04-02T04:46:52.450 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-04-02T04:46:52.530 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-04-02T04:46:52.584 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 8/33 2024-04-02T04:46:52.623 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 9/33 2024-04-02T04:46:52.671 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-ply-3.11-14.el9.noarch 10/33 2024-04-02T04:46:52.737 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 11/33 2024-04-02T04:46:52.920 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 12/33 2024-04-02T04:46:52.968 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 13/33 2024-04-02T04:46:53.012 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 14/33 2024-04-02T04:46:53.050 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 15/33 2024-04-02T04:46:53.080 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 16/33 2024-04-02T04:46:53.157 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 17/33 2024-04-02T04:46:53.190 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-02T04:46:53.223 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 19/33 2024-04-02T04:46:53.263 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 20/33 2024-04-02T04:46:53.298 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 21/33 2024-04-02T04:46:53.341 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 22/33 2024-04-02T04:46:53.431 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 23/33 2024-04-02T04:46:53.472 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 24/33 2024-04-02T04:46:53.553 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 25/33 2024-04-02T04:46:53.691 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 26/33 2024-04-02T04:46:53.816 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-02T04:46:53.971 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-803.gc924f991.el9.n 28/33 2024-04-02T04:46:54.001 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-2:17.2.7-803.gc924f991.el9.x86_64 29/33 2024-04-02T04:46:54.029 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mgr-2:17.2.7-803.gc924f991.el9.x86_64 29/33 2024-04-02T04:46:54.030 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-02T04:46:54.030 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-04-02T04:46:54.030 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-02T04:46:54.030 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-02T04:46:54.030 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:46:54.225 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-test-2:17.2.7-803.gc924f991.el9.x86_64 6/6 2024-04-02T04:46:54.225 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-test-2:17.2.7-803.gc924f991.el9.x86_64 1/6 2024-04-02T04:46:54.226 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : libcephsqlite-2:17.2.7-803.gc924f991.el9.x86_64 2/6 2024-04-02T04:46:54.226 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-04-02T04:46:54.226 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 4/6 2024-04-02T04:46:54.226 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/6 2024-04-02T04:46:54.584 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : jq-1.6-16.el9.x86_64 6/6 2024-04-02T04:46:54.584 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:46:54.584 INFO:teuthology.orchestra.run.smithi154.stdout:Installed: 2024-04-02T04:46:54.584 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-test-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:54.584 INFO:teuthology.orchestra.run.smithi154.stdout: jq-1.6-16.el9.x86_64 2024-04-02T04:46:54.584 INFO:teuthology.orchestra.run.smithi154.stdout: libcephsqlite-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:54.584 INFO:teuthology.orchestra.run.smithi154.stdout: oniguruma-6.9.6-1.el9.5.x86_64 2024-04-02T04:46:54.584 INFO:teuthology.orchestra.run.smithi154.stdout: socat-1.7.4.1-5.el9.x86_64 2024-04-02T04:46:54.584 INFO:teuthology.orchestra.run.smithi154.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-04-02T04:46:54.584 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:46:54.584 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-04-02T04:46:54.882 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum -y install ceph 2024-04-02T04:46:55.336 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-osd-2:17.2.7-803.gc924f991.el9.x86_64 30/33 2024-04-02T04:46:55.363 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-osd-2:17.2.7-803.gc924f991.el9.x86_64 30/33 2024-04-02T04:46:55.363 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-02T04:46:55.363 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-04-02T04:46:55.363 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-02T04:46:55.363 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-02T04:46:55.363 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:46:55.443 INFO:teuthology.orchestra.run.smithi154.stdout:Last metadata expiration check: 0:00:40 ago on Tue 02 Apr 2024 04:46:15 AM UTC. 2024-04-02T04:46:55.597 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-04-02T04:46:55.599 INFO:teuthology.orchestra.run.smithi154.stdout:======================================================================================== 2024-04-02T04:46:55.599 INFO:teuthology.orchestra.run.smithi154.stdout: Package Arch Version Repository Size 2024-04-02T04:46:55.599 INFO:teuthology.orchestra.run.smithi154.stdout:======================================================================================== 2024-04-02T04:46:55.599 INFO:teuthology.orchestra.run.smithi154.stdout:Installing: 2024-04-02T04:46:55.599 INFO:teuthology.orchestra.run.smithi154.stdout: ceph x86_64 2:17.2.7-803.gc924f991.el9 ceph 6.5 k 2024-04-02T04:46:55.599 INFO:teuthology.orchestra.run.smithi154.stdout:Installing dependencies: 2024-04-02T04:46:55.599 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mds x86_64 2:17.2.7-803.gc924f991.el9 ceph 2.1 M 2024-04-02T04:46:55.599 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mgr x86_64 2:17.2.7-803.gc924f991.el9 ceph 1.4 M 2024-04-02T04:46:55.600 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mgr-modules-core noarch 2:17.2.7-803.gc924f991.el9 ceph-noarch 240 k 2024-04-02T04:46:55.600 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mon x86_64 2:17.2.7-803.gc924f991.el9 ceph 3.9 M 2024-04-02T04:46:55.600 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-osd x86_64 2:17.2.7-803.gc924f991.el9 ceph 15 M 2024-04-02T04:46:55.600 INFO:teuthology.orchestra.run.smithi154.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-04-02T04:46:55.600 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cffi x86_64 1.14.5-5.el9 CentOS-BaseOS 241 k 2024-04-02T04:46:55.600 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-04-02T04:46:55.600 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-04-02T04:46:55.600 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cryptography x86_64 36.0.1-4.el9 CentOS-BaseOS 1.1 M 2024-04-02T04:46:55.600 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-04-02T04:46:55.600 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-04-02T04:46:55.600 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-04-02T04:46:55.600 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-04-02T04:46:55.600 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-04-02T04:46:55.600 INFO:teuthology.orchestra.run.smithi154.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-04-02T04:46:55.600 INFO:teuthology.orchestra.run.smithi154.stdout: python3-mako noarch 1.1.4-6.el9 CentOS-AppStream 144 k 2024-04-02T04:46:55.601 INFO:teuthology.orchestra.run.smithi154.stdout: python3-markupsafe x86_64 1.1.1-12.el9 CentOS-AppStream 32 k 2024-04-02T04:46:55.601 INFO:teuthology.orchestra.run.smithi154.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-04-02T04:46:55.601 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-04-02T04:46:55.601 INFO:teuthology.orchestra.run.smithi154.stdout: python3-ply noarch 3.11-14.el9 CentOS-BaseOS 103 k 2024-04-02T04:46:55.601 INFO:teuthology.orchestra.run.smithi154.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-04-02T04:46:55.601 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-04-02T04:46:55.601 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pycparser noarch 2.20-6.el9 CentOS-BaseOS 124 k 2024-04-02T04:46:55.601 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pytz noarch 2021.1-5.el9 CentOS-AppStream 47 k 2024-04-02T04:46:55.601 INFO:teuthology.orchestra.run.smithi154.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-04-02T04:46:55.601 INFO:teuthology.orchestra.run.smithi154.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-04-02T04:46:55.601 INFO:teuthology.orchestra.run.smithi154.stdout: python3-toml noarch 0.10.2-6.el9 CentOS-AppStream 39 k 2024-04-02T04:46:55.601 INFO:teuthology.orchestra.run.smithi154.stdout: python3-urllib3 noarch 1.26.5-5.el9 CentOS-BaseOS 187 k 2024-04-02T04:46:55.601 INFO:teuthology.orchestra.run.smithi154.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-04-02T04:46:55.601 INFO:teuthology.orchestra.run.smithi154.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-04-02T04:46:55.601 INFO:teuthology.orchestra.run.smithi154.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-04-02T04:46:55.602 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:46:55.602 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction Summary 2024-04-02T04:46:55.602 INFO:teuthology.orchestra.run.smithi154.stdout:======================================================================================== 2024-04-02T04:46:55.602 INFO:teuthology.orchestra.run.smithi154.stdout:Install 33 Packages 2024-04-02T04:46:55.602 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:46:55.602 INFO:teuthology.orchestra.run.smithi154.stdout:Total download size: 27 M 2024-04-02T04:46:55.602 INFO:teuthology.orchestra.run.smithi154.stdout:Installed size: 96 M 2024-04-02T04:46:55.603 INFO:teuthology.orchestra.run.smithi154.stdout:Downloading Packages: 2024-04-02T04:46:55.728 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mon-2:17.2.7-803.gc924f991.el9.x86_64 31/33 2024-04-02T04:46:55.752 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mon-2:17.2.7-803.gc924f991.el9.x86_64 31/33 2024-04-02T04:46:55.752 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-02T04:46:55.752 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-04-02T04:46:55.752 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-02T04:46:55.752 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-02T04:46:55.752 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:46:55.965 INFO:teuthology.orchestra.run.smithi154.stdout:(1/33): ceph-17.2.7-803.gc924f991.el9.x86_64.rp 39 kB/s | 6.5 kB 00:00 2024-04-02T04:46:55.987 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mds-2:17.2.7-803.gc924f991.el9.x86_64 32/33 2024-04-02T04:46:56.011 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mds-2:17.2.7-803.gc924f991.el9.x86_64 32/33 2024-04-02T04:46:56.011 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-02T04:46:56.011 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-04-02T04:46:56.011 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-02T04:46:56.012 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-02T04:46:56.012 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:46:56.077 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-2:17.2.7-803.gc924f991.el9.x86_64 33/33 2024-04-02T04:46:56.157 INFO:teuthology.orchestra.run.smithi154.stdout:(2/33): ceph-mgr-17.2.7-803.gc924f991.el9.x86_6 3.9 MB/s | 1.4 MB 00:00 2024-04-02T04:46:56.449 INFO:teuthology.orchestra.run.smithi154.stdout:(3/33): ceph-mon-17.2.7-803.gc924f991.el9.x86_6 8.1 MB/s | 3.9 MB 00:00 2024-04-02T04:46:56.525 INFO:teuthology.orchestra.run.smithi154.stdout:(4/33): ceph-mds-17.2.7-803.gc924f991.el9.x86_6 2.8 MB/s | 2.1 MB 00:00 2024-04-02T04:46:56.550 INFO:teuthology.orchestra.run.smithi154.stdout:(5/33): ceph-mgr-modules-core-17.2.7-803.gc924f 2.3 MB/s | 240 kB 00:00 2024-04-02T04:46:56.667 INFO:teuthology.orchestra.run.smithi154.stdout:(6/33): python3-markupsafe-1.1.1-12.el9.x86_64. 271 kB/s | 32 kB 00:00 2024-04-02T04:46:56.692 INFO:teuthology.orchestra.run.smithi154.stdout:(7/33): python3-mako-1.1.4-6.el9.noarch.rpm 864 kB/s | 144 kB 00:00 2024-04-02T04:46:56.717 INFO:teuthology.orchestra.run.smithi154.stdout:(8/33): python3-pytz-2021.1-5.el9.noarch.rpm 944 kB/s | 47 kB 00:00 2024-04-02T04:46:56.743 INFO:teuthology.orchestra.run.smithi154.stdout:(9/33): python3-toml-0.10.2-6.el9.noarch.rpm 769 kB/s | 39 kB 00:00 2024-04-02T04:46:56.777 INFO:teuthology.orchestra.run.smithi154.stdout:(10/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 4.0 MB/s | 241 kB 00:00 2024-04-02T04:46:56.835 INFO:teuthology.orchestra.run.smithi154.stdout:(11/33): python3-cryptography-36.0.1-4.el9.x86_ 12 MB/s | 1.1 MB 00:00 2024-04-02T04:46:56.860 INFO:teuthology.orchestra.run.smithi154.stdout:(12/33): python3-ply-3.11-14.el9.noarch.rpm 1.2 MB/s | 103 kB 00:00 2024-04-02T04:46:56.886 INFO:teuthology.orchestra.run.smithi154.stdout:(13/33): python3-pycparser-2.20-6.el9.noarch.rp 2.4 MB/s | 124 kB 00:00 2024-04-02T04:46:56.911 INFO:teuthology.orchestra.run.smithi154.stdout:(14/33): python3-urllib3-1.26.5-5.el9.noarch.rp 3.6 MB/s | 187 kB 00:00 2024-04-02T04:46:57.111 INFO:teuthology.orchestra.run.smithi154.stdout:(15/33): ceph-osd-17.2.7-803.gc924f991.el9.x86_ 16 MB/s | 15 MB 00:00 2024-04-02T04:46:57.270 INFO:teuthology.orchestra.run.smithi154.stdout:(16/33): python3-requests-2.25.1-8.el9.noarch.r 326 kB/s | 125 kB 00:00 2024-04-02T04:46:57.370 INFO:teuthology.orchestra.run.smithi154.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 95 kB/s | 43 kB 00:00 2024-04-02T04:46:57.478 INFO:teuthology.orchestra.run.smithi154.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 99 kB/s | 11 kB 00:00 2024-04-02T04:46:57.596 INFO:teuthology.orchestra.run.smithi154.stdout:(19/33): python3-jaraco-classes-3.2.1-5.el9.noa 151 kB/s | 18 kB 00:00 2024-04-02T04:46:57.621 INFO:teuthology.orchestra.run.smithi154.stdout:(20/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 337 kB/s | 172 kB 00:00 2024-04-02T04:46:57.710 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-2:17.2.7-803.gc924f991.el9.x86_64 33/33 2024-04-02T04:46:57.711 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-2:17.2.7-803.gc924f991.el9.x86_64 1/33 2024-04-02T04:46:57.711 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mds-2:17.2.7-803.gc924f991.el9.x86_64 2/33 2024-04-02T04:46:57.711 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-2:17.2.7-803.gc924f991.el9.x86_64 3/33 2024-04-02T04:46:57.711 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mon-2:17.2.7-803.gc924f991.el9.x86_64 4/33 2024-04-02T04:46:57.711 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-osd-2:17.2.7-803.gc924f991.el9.x86_64 5/33 2024-04-02T04:46:57.711 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-803.gc924f991.el9.n 6/33 2024-04-02T04:46:57.711 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 7/33 2024-04-02T04:46:57.711 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 8/33 2024-04-02T04:46:57.711 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 9/33 2024-04-02T04:46:57.711 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 10/33 2024-04-02T04:46:57.711 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 11/33 2024-04-02T04:46:57.712 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 12/33 2024-04-02T04:46:57.712 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-ply-3.11-14.el9.noarch 13/33 2024-04-02T04:46:57.712 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 14/33 2024-04-02T04:46:57.712 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 15/33 2024-04-02T04:46:57.712 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 16/33 2024-04-02T04:46:57.712 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-04-02T04:46:57.712 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-02T04:46:57.712 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-04-02T04:46:57.712 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-04-02T04:46:57.712 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-04-02T04:46:57.712 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-04-02T04:46:57.712 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-04-02T04:46:57.712 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-04-02T04:46:57.712 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-04-02T04:46:57.712 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-04-02T04:46:57.713 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-02T04:46:57.713 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-04-02T04:46:57.713 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-04-02T04:46:57.713 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-04-02T04:46:57.713 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-04-02T04:46:57.713 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-04-02T04:46:57.713 INFO:teuthology.orchestra.run.smithi154.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 198 kB/s | 23 kB 00:00 2024-04-02T04:46:57.738 INFO:teuthology.orchestra.run.smithi154.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 166 kB/s | 19 kB 00:00 2024-04-02T04:46:57.830 INFO:teuthology.orchestra.run.smithi154.stdout:(23/33): python3-jaraco-text-3.2.0-6.el9.noarch 168 kB/s | 20 kB 00:00 2024-04-02T04:46:57.855 INFO:teuthology.orchestra.run.smithi154.stdout:(24/33): python3-logutils-0.3.5-21.el9.noarch.r 396 kB/s | 46 kB 00:00 2024-04-02T04:46:57.897 INFO:teuthology.orchestra.run.smithi154.stdout:(25/33): python3-cherrypy-18.6.1-2.el9.noarch.r 572 kB/s | 358 kB 00:00 2024-04-02T04:46:57.997 INFO:teuthology.orchestra.run.smithi154.stdout:(26/33): python3-portend-3.1.0-2.el9.noarch.rpm 164 kB/s | 16 kB 00:00 2024-04-02T04:46:58.022 INFO:teuthology.orchestra.run.smithi154.stdout:(27/33): python3-more-itertools-8.12.0-2.el9.no 410 kB/s | 79 kB 00:00 2024-04-02T04:46:58.056 INFO:teuthology.orchestra.run.smithi154.stdout:(28/33): python3-pecan-1.4.2-3.el9.noarch.rpm 1.3 MB/s | 272 kB 00:00 2024-04-02T04:46:58.131 INFO:teuthology.orchestra.run.smithi154.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 330 kB/s | 36 kB 00:00 2024-04-02T04:46:58.156 INFO:teuthology.orchestra.run.smithi154.stdout:(30/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 566 kB/s | 90 kB 00:00 2024-04-02T04:46:58.182 INFO:teuthology.orchestra.run.smithi154.stdout:(31/33): python3-webob-1.8.7-6.el9.noarch.rpm 1.8 MB/s | 230 kB 00:00 2024-04-02T04:46:58.265 INFO:teuthology.orchestra.run.smithi154.stdout:(32/33): python3-zc-lockfile-2.0-10.el9.noarch. 184 kB/s | 20 kB 00:00 2024-04-02T04:46:58.315 INFO:teuthology.orchestra.run.smithi154.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 2.3 MB/s | 427 kB 00:00 2024-04-02T04:46:58.319 INFO:teuthology.orchestra.run.smithi154.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:46:58.319 INFO:teuthology.orchestra.run.smithi154.stdout:Total 9.9 MB/s | 27 MB 00:02 2024-04-02T04:46:58.334 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction check 2024-04-02T04:46:58.375 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction check succeeded. 2024-04-02T04:46:58.375 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction test 2024-04-02T04:46:58.557 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-04-02T04:46:58.557 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:46:58.557 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-02T04:46:58.557 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:58.557 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mds-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:58.557 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:58.557 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-modules-core-2:17.2.7-803.gc924f991.el9.noarch 2024-04-02T04:46:58.557 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mon-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:58.557 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-osd-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:46:58.557 INFO:teuthology.orchestra.run.smithi152.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-04-02T04:46:58.557 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-04-02T04:46:58.558 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-04-02T04:46:58.558 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-04-02T04:46:58.558 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-04-02T04:46:58.558 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-04-02T04:46:58.558 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-04-02T04:46:58.558 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-04-02T04:46:58.558 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-04-02T04:46:58.558 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-04-02T04:46:58.558 INFO:teuthology.orchestra.run.smithi152.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-04-02T04:46:58.558 INFO:teuthology.orchestra.run.smithi152.stdout: python3-mako-1.1.4-6.el9.noarch 2024-04-02T04:46:58.558 INFO:teuthology.orchestra.run.smithi152.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-04-02T04:46:58.558 INFO:teuthology.orchestra.run.smithi152.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-04-02T04:46:58.558 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-04-02T04:46:58.558 INFO:teuthology.orchestra.run.smithi152.stdout: python3-ply-3.11-14.el9.noarch 2024-04-02T04:46:58.559 INFO:teuthology.orchestra.run.smithi152.stdout: python3-portend-3.1.0-2.el9.noarch 2024-04-02T04:46:58.559 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-04-02T04:46:58.559 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pycparser-2.20-6.el9.noarch 2024-04-02T04:46:58.559 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pytz-2021.1-5.el9.noarch 2024-04-02T04:46:58.559 INFO:teuthology.orchestra.run.smithi152.stdout: python3-requests-2.25.1-8.el9.noarch 2024-04-02T04:46:58.559 INFO:teuthology.orchestra.run.smithi152.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-04-02T04:46:58.559 INFO:teuthology.orchestra.run.smithi152.stdout: python3-toml-0.10.2-6.el9.noarch 2024-04-02T04:46:58.559 INFO:teuthology.orchestra.run.smithi152.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-04-02T04:46:58.559 INFO:teuthology.orchestra.run.smithi152.stdout: python3-webob-1.8.7-6.el9.noarch 2024-04-02T04:46:58.559 INFO:teuthology.orchestra.run.smithi152.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-04-02T04:46:58.559 INFO:teuthology.orchestra.run.smithi152.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-04-02T04:46:58.559 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:46:58.559 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-02T04:46:58.718 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction test succeeded. 2024-04-02T04:46:58.719 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction 2024-04-02T04:46:58.922 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-base 2024-04-02T04:46:59.492 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:53 ago on Tue 02 Apr 2024 04:46:06 AM UTC. 2024-04-02T04:46:59.590 INFO:teuthology.orchestra.run.smithi152.stdout:Package ceph-base-2:17.2.7-803.gc924f991.el9.x86_64 is already installed. 2024-04-02T04:46:59.641 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-02T04:46:59.642 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-02T04:46:59.642 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-02T04:46:59.717 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install cephadm 2024-04-02T04:46:59.970 INFO:teuthology.orchestra.run.smithi154.stdout: Preparing : 1/1 2024-04-02T04:47:00.044 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-04-02T04:47:00.075 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-04-02T04:47:00.117 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-04-02T04:47:00.209 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-04-02T04:47:00.279 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:54 ago on Tue 02 Apr 2024 04:46:06 AM UTC. 2024-04-02T04:47:00.287 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-04-02T04:47:00.328 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-04-02T04:47:00.400 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-04-02T04:47:00.430 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-02T04:47:00.431 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:00.431 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-02T04:47:00.431 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:00.431 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-02T04:47:00.432 INFO:teuthology.orchestra.run.smithi152.stdout: cephadm noarch 2:17.2.7-803.gc924f991.el9 ceph-noarch 82 k 2024-04-02T04:47:00.432 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:00.432 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-02T04:47:00.432 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:00.432 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-04-02T04:47:00.432 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:00.432 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 82 k 2024-04-02T04:47:00.432 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 372 k 2024-04-02T04:47:00.432 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-02T04:47:00.454 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 8/33 2024-04-02T04:47:00.492 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 9/33 2024-04-02T04:47:00.540 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-ply-3.11-14.el9.noarch 10/33 2024-04-02T04:47:00.607 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 11/33 2024-04-02T04:47:00.652 INFO:teuthology.orchestra.run.smithi152.stdout:cephadm-17.2.7-803.gc924f991.el9.noarch.rpm 374 kB/s | 82 kB 00:00 2024-04-02T04:47:00.652 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:47:00.652 INFO:teuthology.orchestra.run.smithi152.stdout:Total 371 kB/s | 82 kB 00:00 2024-04-02T04:47:00.653 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-02T04:47:00.657 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-02T04:47:00.658 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-02T04:47:00.659 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-02T04:47:00.659 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-02T04:47:00.735 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-02T04:47:00.789 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 12/33 2024-04-02T04:47:00.846 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 13/33 2024-04-02T04:47:00.889 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 14/33 2024-04-02T04:47:00.928 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 15/33 2024-04-02T04:47:00.957 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 16/33 2024-04-02T04:47:01.033 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 17/33 2024-04-02T04:47:01.067 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-02T04:47:01.101 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 19/33 2024-04-02T04:47:01.141 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 20/33 2024-04-02T04:47:01.158 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: cephadm-2:17.2.7-803.gc924f991.el9.noarch 1/1 2024-04-02T04:47:01.176 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 21/33 2024-04-02T04:47:01.210 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 22/33 2024-04-02T04:47:01.238 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : cephadm-2:17.2.7-803.gc924f991.el9.noarch 1/1 2024-04-02T04:47:01.300 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 23/33 2024-04-02T04:47:01.341 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 24/33 2024-04-02T04:47:01.413 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 25/33 2024-04-02T04:47:01.550 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 26/33 2024-04-02T04:47:01.603 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: cephadm-2:17.2.7-803.gc924f991.el9.noarch 1/1 2024-04-02T04:47:01.675 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-02T04:47:01.822 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-803.gc924f991.el9.n 28/33 2024-04-02T04:47:01.863 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-mgr-2:17.2.7-803.gc924f991.el9.x86_64 29/33 2024-04-02T04:47:01.872 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : cephadm-2:17.2.7-803.gc924f991.el9.noarch 1/1 2024-04-02T04:47:01.872 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:01.872 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-02T04:47:01.872 INFO:teuthology.orchestra.run.smithi152.stdout: cephadm-2:17.2.7-803.gc924f991.el9.noarch 2024-04-02T04:47:01.872 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:01.872 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-02T04:47:01.887 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-mgr-2:17.2.7-803.gc924f991.el9.x86_64 29/33 2024-04-02T04:47:01.888 INFO:teuthology.orchestra.run.smithi154.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-02T04:47:01.888 INFO:teuthology.orchestra.run.smithi154.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-04-02T04:47:01.888 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-02T04:47:01.888 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-04-02T04:47:01.888 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:02.065 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-immutable-object-cache 2024-04-02T04:47:02.640 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:56 ago on Tue 02 Apr 2024 04:46:06 AM UTC. 2024-04-02T04:47:02.791 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-02T04:47:02.792 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:02.792 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repo Size 2024-04-02T04:47:02.792 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:02.792 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-02T04:47:02.792 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-803.gc924f991.el9 ceph 137 k 2024-04-02T04:47:02.792 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:02.792 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-02T04:47:02.792 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:02.792 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-04-02T04:47:02.793 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:02.793 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 137 k 2024-04-02T04:47:02.793 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 379 k 2024-04-02T04:47:02.793 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-02T04:47:03.087 INFO:teuthology.orchestra.run.smithi152.stdout:ceph-immutable-object-cache-17.2.7-803.gc924f99 466 kB/s | 137 kB 00:00 2024-04-02T04:47:03.088 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:47:03.088 INFO:teuthology.orchestra.run.smithi152.stdout:Total 464 kB/s | 137 kB 00:00 2024-04-02T04:47:03.088 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-02T04:47:03.095 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-02T04:47:03.095 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-02T04:47:03.146 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-02T04:47:03.147 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-02T04:47:03.223 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-osd-2:17.2.7-803.gc924f991.el9.x86_64 30/33 2024-04-02T04:47:03.252 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-osd-2:17.2.7-803.gc924f991.el9.x86_64 30/33 2024-04-02T04:47:03.252 INFO:teuthology.orchestra.run.smithi154.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-02T04:47:03.252 INFO:teuthology.orchestra.run.smithi154.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-04-02T04:47:03.252 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-02T04:47:03.252 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-04-02T04:47:03.252 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:03.285 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-02T04:47:03.338 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-803.gc924f991.e 1/1 2024-04-02T04:47:03.362 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-803.gc924f991.e 1/1 2024-04-02T04:47:03.363 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-02T04:47:03.363 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-04-02T04:47:03.363 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:03.648 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-mon-2:17.2.7-803.gc924f991.el9.x86_64 31/33 2024-04-02T04:47:03.673 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-mon-2:17.2.7-803.gc924f991.el9.x86_64 31/33 2024-04-02T04:47:03.674 INFO:teuthology.orchestra.run.smithi154.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-02T04:47:03.674 INFO:teuthology.orchestra.run.smithi154.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-04-02T04:47:03.674 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-02T04:47:03.674 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-04-02T04:47:03.674 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:03.891 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-mds-2:17.2.7-803.gc924f991.el9.x86_64 32/33 2024-04-02T04:47:03.917 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-mds-2:17.2.7-803.gc924f991.el9.x86_64 32/33 2024-04-02T04:47:03.917 INFO:teuthology.orchestra.run.smithi154.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-02T04:47:03.917 INFO:teuthology.orchestra.run.smithi154.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-04-02T04:47:03.918 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-02T04:47:03.918 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-04-02T04:47:03.918 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:03.981 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-2:17.2.7-803.gc924f991.el9.x86_64 33/33 2024-04-02T04:47:04.349 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-803.gc924f991.e 1/1 2024-04-02T04:47:04.349 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:04.349 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-02T04:47:04.349 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-immutable-object-cache-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:47:04.349 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:04.350 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-02T04:47:04.618 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-mgr 2024-04-02T04:47:05.196 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:59 ago on Tue 02 Apr 2024 04:46:06 AM UTC. 2024-04-02T04:47:05.295 INFO:teuthology.orchestra.run.smithi152.stdout:Package ceph-mgr-2:17.2.7-803.gc924f991.el9.x86_64 is already installed. 2024-04-02T04:47:05.347 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-02T04:47:05.347 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-02T04:47:05.348 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-02T04:47:05.423 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-mgr-dashboard 2024-04-02T04:47:05.655 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-2:17.2.7-803.gc924f991.el9.x86_64 33/33 2024-04-02T04:47:05.655 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-2:17.2.7-803.gc924f991.el9.x86_64 1/33 2024-04-02T04:47:05.655 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-mds-2:17.2.7-803.gc924f991.el9.x86_64 2/33 2024-04-02T04:47:05.656 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-mgr-2:17.2.7-803.gc924f991.el9.x86_64 3/33 2024-04-02T04:47:05.656 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-mon-2:17.2.7-803.gc924f991.el9.x86_64 4/33 2024-04-02T04:47:05.656 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-osd-2:17.2.7-803.gc924f991.el9.x86_64 5/33 2024-04-02T04:47:05.656 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-803.gc924f991.el9.n 6/33 2024-04-02T04:47:05.656 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 7/33 2024-04-02T04:47:05.656 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 8/33 2024-04-02T04:47:05.656 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 9/33 2024-04-02T04:47:05.656 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 10/33 2024-04-02T04:47:05.656 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 11/33 2024-04-02T04:47:05.656 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 12/33 2024-04-02T04:47:05.656 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-ply-3.11-14.el9.noarch 13/33 2024-04-02T04:47:05.656 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 14/33 2024-04-02T04:47:05.656 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 15/33 2024-04-02T04:47:05.656 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 16/33 2024-04-02T04:47:05.656 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-04-02T04:47:05.657 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-04-02T04:47:05.657 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-04-02T04:47:05.657 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-04-02T04:47:05.657 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-04-02T04:47:05.657 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-04-02T04:47:05.657 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-04-02T04:47:05.657 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-04-02T04:47:05.657 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-04-02T04:47:05.657 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-04-02T04:47:05.657 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-04-02T04:47:05.657 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-04-02T04:47:05.658 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-04-02T04:47:05.658 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-04-02T04:47:05.658 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-04-02T04:47:05.658 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-04-02T04:47:05.979 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:00:59 ago on Tue 02 Apr 2024 04:46:06 AM UTC. 2024-04-02T04:47:06.128 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-02T04:47:06.129 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:06.129 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-02T04:47:06.129 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:06.129 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-02T04:47:06.129 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-dashboard noarch 2:17.2.7-803.gc924f991.el9 ceph-noarch 1.7 M 2024-04-02T04:47:06.130 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-02T04:47:06.130 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-grafana-dashboards noarch 2:17.2.7-803.gc924f991.el9 ceph-noarch 23 k 2024-04-02T04:47:06.130 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-prometheus-alerts noarch 2:17.2.7-803.gc924f991.el9 ceph-noarch 15 k 2024-04-02T04:47:06.130 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-04-02T04:47:06.130 INFO:teuthology.orchestra.run.smithi152.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-04-02T04:47:06.130 INFO:teuthology.orchestra.run.smithi152.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-04-02T04:47:06.130 INFO:teuthology.orchestra.run.smithi152.stdout:Installing weak dependencies: 2024-04-02T04:47:06.130 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-04-02T04:47:06.130 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:06.130 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-02T04:47:06.130 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:06.130 INFO:teuthology.orchestra.run.smithi152.stdout:Install 7 Packages 2024-04-02T04:47:06.130 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:06.131 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 2.0 M 2024-04-02T04:47:06.131 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 8.1 M 2024-04-02T04:47:06.131 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-02T04:47:06.441 INFO:teuthology.orchestra.run.smithi152.stdout:(1/7): ceph-prometheus-alerts-17.2.7-803.gc924f 57 kB/s | 15 kB 00:00 2024-04-02T04:47:06.466 INFO:teuthology.orchestra.run.smithi152.stdout:(2/7): ceph-grafana-dashboards-17.2.7-803.gc924 83 kB/s | 23 kB 00:00 2024-04-02T04:47:06.567 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-04-02T04:47:06.568 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:06.568 INFO:teuthology.orchestra.run.smithi154.stdout:Installed: 2024-04-02T04:47:06.568 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:47:06.568 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mds-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:47:06.568 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mgr-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:47:06.568 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mgr-modules-core-2:17.2.7-803.gc924f991.el9.noarch 2024-04-02T04:47:06.568 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mon-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:47:06.568 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-osd-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:47:06.568 INFO:teuthology.orchestra.run.smithi154.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-04-02T04:47:06.568 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-04-02T04:47:06.568 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-04-02T04:47:06.568 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-04-02T04:47:06.568 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-04-02T04:47:06.568 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-04-02T04:47:06.568 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-04-02T04:47:06.569 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-04-02T04:47:06.569 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-04-02T04:47:06.569 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-04-02T04:47:06.569 INFO:teuthology.orchestra.run.smithi154.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-04-02T04:47:06.569 INFO:teuthology.orchestra.run.smithi154.stdout: python3-mako-1.1.4-6.el9.noarch 2024-04-02T04:47:06.569 INFO:teuthology.orchestra.run.smithi154.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-04-02T04:47:06.569 INFO:teuthology.orchestra.run.smithi154.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-04-02T04:47:06.569 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-04-02T04:47:06.569 INFO:teuthology.orchestra.run.smithi154.stdout: python3-ply-3.11-14.el9.noarch 2024-04-02T04:47:06.569 INFO:teuthology.orchestra.run.smithi154.stdout: python3-portend-3.1.0-2.el9.noarch 2024-04-02T04:47:06.569 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-04-02T04:47:06.569 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pycparser-2.20-6.el9.noarch 2024-04-02T04:47:06.569 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pytz-2021.1-5.el9.noarch 2024-04-02T04:47:06.569 INFO:teuthology.orchestra.run.smithi154.stdout: python3-requests-2.25.1-8.el9.noarch 2024-04-02T04:47:06.569 INFO:teuthology.orchestra.run.smithi154.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-04-02T04:47:06.569 INFO:teuthology.orchestra.run.smithi154.stdout: python3-toml-0.10.2-6.el9.noarch 2024-04-02T04:47:06.570 INFO:teuthology.orchestra.run.smithi154.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-04-02T04:47:06.570 INFO:teuthology.orchestra.run.smithi154.stdout: python3-webob-1.8.7-6.el9.noarch 2024-04-02T04:47:06.570 INFO:teuthology.orchestra.run.smithi154.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-04-02T04:47:06.570 INFO:teuthology.orchestra.run.smithi154.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-04-02T04:47:06.570 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:06.570 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-04-02T04:47:06.599 INFO:teuthology.orchestra.run.smithi152.stdout:(3/7): ceph-mgr-dashboard-17.2.7-803.gc924f991. 4.1 MB/s | 1.7 MB 00:00 2024-04-02T04:47:06.616 INFO:teuthology.orchestra.run.smithi152.stdout:(4/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 52 kB/s | 9.0 kB 00:00 2024-04-02T04:47:06.650 INFO:teuthology.orchestra.run.smithi152.stdout:(5/7): python3-jwt-2.4.0-1.el9.noarch.rpm 223 kB/s | 41 kB 00:00 2024-04-02T04:47:06.667 INFO:teuthology.orchestra.run.smithi152.stdout:(6/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 460 kB/s | 31 kB 00:00 2024-04-02T04:47:06.733 INFO:teuthology.orchestra.run.smithi152.stdout:(7/7): python3-routes-2.5.1-5.el9.noarch.rpm 1.6 MB/s | 188 kB 00:00 2024-04-02T04:47:06.737 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:47:06.737 INFO:teuthology.orchestra.run.smithi152.stdout:Total 3.3 MB/s | 2.0 MB 00:00 2024-04-02T04:47:06.737 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-02T04:47:06.744 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-02T04:47:06.744 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-02T04:47:06.803 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-02T04:47:06.804 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-02T04:47:06.938 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum -y install ceph-base 2024-04-02T04:47:07.108 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-02T04:47:07.177 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-04-02T04:47:07.211 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-04-02T04:47:07.257 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-04-02T04:47:07.289 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-04-02T04:47:07.318 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-803.gc924f991.el9.no 5/7 2024-04-02T04:47:07.505 INFO:teuthology.orchestra.run.smithi154.stdout:Last metadata expiration check: 0:00:52 ago on Tue 02 Apr 2024 04:46:15 AM UTC. 2024-04-02T04:47:07.546 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-803.gc924f991.el9.n 6/7 2024-04-02T04:47:07.579 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-803.gc924f991.el9.noarch 7/7 2024-04-02T04:47:07.603 INFO:teuthology.orchestra.run.smithi154.stdout:Package ceph-base-2:17.2.7-803.gc924f991.el9.x86_64 is already installed. 2024-04-02T04:47:07.654 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-04-02T04:47:07.655 INFO:teuthology.orchestra.run.smithi154.stdout:Nothing to do. 2024-04-02T04:47:07.655 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-04-02T04:47:07.725 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum -y install cephadm 2024-04-02T04:47:08.120 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-803.gc924f991.el9.noarch 7/7 2024-04-02T04:47:08.120 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-803.gc924f991.el9.n 1/7 2024-04-02T04:47:08.120 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-803.gc924f991.el9.noarch 2/7 2024-04-02T04:47:08.120 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-803.gc924f991.el9.no 3/7 2024-04-02T04:47:08.120 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-04-02T04:47:08.120 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-04-02T04:47:08.120 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-04-02T04:47:08.286 INFO:teuthology.orchestra.run.smithi154.stdout:Last metadata expiration check: 0:00:53 ago on Tue 02 Apr 2024 04:46:15 AM UTC. 2024-04-02T04:47:08.413 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-04-02T04:47:08.413 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:08.413 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-02T04:47:08.413 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-grafana-dashboards-2:17.2.7-803.gc924f991.el9.noarch 2024-04-02T04:47:08.413 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-dashboard-2:17.2.7-803.gc924f991.el9.noarch 2024-04-02T04:47:08.413 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-prometheus-alerts-2:17.2.7-803.gc924f991.el9.noarch 2024-04-02T04:47:08.413 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-04-02T04:47:08.413 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-04-02T04:47:08.413 INFO:teuthology.orchestra.run.smithi152.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-04-02T04:47:08.413 INFO:teuthology.orchestra.run.smithi152.stdout: python3-routes-2.5.1-5.el9.noarch 2024-04-02T04:47:08.413 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:08.414 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-02T04:47:08.435 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-04-02T04:47:08.436 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:08.436 INFO:teuthology.orchestra.run.smithi154.stdout: Package Arch Version Repository Size 2024-04-02T04:47:08.436 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:08.436 INFO:teuthology.orchestra.run.smithi154.stdout:Installing: 2024-04-02T04:47:08.436 INFO:teuthology.orchestra.run.smithi154.stdout: cephadm noarch 2:17.2.7-803.gc924f991.el9 ceph-noarch 82 k 2024-04-02T04:47:08.437 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:08.437 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction Summary 2024-04-02T04:47:08.437 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:08.437 INFO:teuthology.orchestra.run.smithi154.stdout:Install 1 Package 2024-04-02T04:47:08.437 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:08.437 INFO:teuthology.orchestra.run.smithi154.stdout:Total download size: 82 k 2024-04-02T04:47:08.437 INFO:teuthology.orchestra.run.smithi154.stdout:Installed size: 372 k 2024-04-02T04:47:08.437 INFO:teuthology.orchestra.run.smithi154.stdout:Downloading Packages: 2024-04-02T04:47:08.673 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-mgr-diskprediction-local 2024-04-02T04:47:08.714 INFO:teuthology.orchestra.run.smithi154.stdout:cephadm-17.2.7-803.gc924f991.el9.noarch.rpm 296 kB/s | 82 kB 00:00 2024-04-02T04:47:08.715 INFO:teuthology.orchestra.run.smithi154.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:47:08.715 INFO:teuthology.orchestra.run.smithi154.stdout:Total 295 kB/s | 82 kB 00:00 2024-04-02T04:47:08.715 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction check 2024-04-02T04:47:08.720 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction check succeeded. 2024-04-02T04:47:08.720 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction test 2024-04-02T04:47:08.722 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction test succeeded. 2024-04-02T04:47:08.723 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction 2024-04-02T04:47:08.797 INFO:teuthology.orchestra.run.smithi154.stdout: Preparing : 1/1 2024-04-02T04:47:09.231 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:03 ago on Tue 02 Apr 2024 04:46:06 AM UTC. 2024-04-02T04:47:09.263 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: cephadm-2:17.2.7-803.gc924f991.el9.noarch 1/1 2024-04-02T04:47:09.361 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : cephadm-2:17.2.7-803.gc924f991.el9.noarch 1/1 2024-04-02T04:47:09.383 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-02T04:47:09.384 INFO:teuthology.orchestra.run.smithi152.stdout:=========================================================================================== 2024-04-02T04:47:09.384 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-02T04:47:09.384 INFO:teuthology.orchestra.run.smithi152.stdout:=========================================================================================== 2024-04-02T04:47:09.384 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-02T04:47:09.384 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-803.gc924f991.el9 ceph-noarch 7.4 M 2024-04-02T04:47:09.384 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-02T04:47:09.384 INFO:teuthology.orchestra.run.smithi152.stdout: flexiblas x86_64 3.0.4-8.el9 CentOS-AppStream 32 k 2024-04-02T04:47:09.384 INFO:teuthology.orchestra.run.smithi152.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 CentOS-AppStream 3.0 M 2024-04-02T04:47:09.384 INFO:teuthology.orchestra.run.smithi152.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 CentOS-AppStream 17 k 2024-04-02T04:47:09.384 INFO:teuthology.orchestra.run.smithi152.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-04-02T04:47:09.384 INFO:teuthology.orchestra.run.smithi152.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-04-02T04:47:09.385 INFO:teuthology.orchestra.run.smithi152.stdout: openblas x86_64 0.3.21-2.el9 CentOS-AppStream 34 k 2024-04-02T04:47:09.385 INFO:teuthology.orchestra.run.smithi152.stdout: openblas-openmp x86_64 0.3.21-2.el9 CentOS-AppStream 4.7 M 2024-04-02T04:47:09.385 INFO:teuthology.orchestra.run.smithi152.stdout: python3-devel x86_64 3.9.17-2.el9 appstream 245 k 2024-04-02T04:47:09.385 INFO:teuthology.orchestra.run.smithi152.stdout: python3-numpy x86_64 1:1.20.1-5.el9 CentOS-AppStream 4.8 M 2024-04-02T04:47:09.385 INFO:teuthology.orchestra.run.smithi152.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 CentOS-AppStream 217 k 2024-04-02T04:47:09.385 INFO:teuthology.orchestra.run.smithi152.stdout: python3-scipy x86_64 1.6.2-8.el9 CentOS-AppStream 16 M 2024-04-02T04:47:09.385 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:09.385 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-02T04:47:09.385 INFO:teuthology.orchestra.run.smithi152.stdout:=========================================================================================== 2024-04-02T04:47:09.385 INFO:teuthology.orchestra.run.smithi152.stdout:Install 12 Packages 2024-04-02T04:47:09.385 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:09.386 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 37 M 2024-04-02T04:47:09.386 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 209 M 2024-04-02T04:47:09.386 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-02T04:47:09.615 INFO:teuthology.orchestra.run.smithi152.stdout:(1/12): flexiblas-3.0.4-8.el9.x86_64.rpm 248 kB/s | 32 kB 00:00 2024-04-02T04:47:09.649 INFO:teuthology.orchestra.run.smithi152.stdout:(2/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 536 kB/s | 17 kB 00:00 2024-04-02T04:47:09.733 INFO:teuthology.orchestra.run.smithi152.stdout:(3/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 12 MB/s | 3.0 MB 00:00 2024-04-02T04:47:09.758 INFO:teuthology.orchestra.run.smithi152.stdout:(4/12): openblas-0.3.21-2.el9.x86_64.rpm 320 kB/s | 34 kB 00:00 2024-04-02T04:47:09.798 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: cephadm-2:17.2.7-803.gc924f991.el9.noarch 1/1 2024-04-02T04:47:09.958 INFO:teuthology.orchestra.run.smithi152.stdout:(5/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 21 MB/s | 4.7 MB 00:00 2024-04-02T04:47:10.025 INFO:teuthology.orchestra.run.smithi152.stdout:(6/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 18 MB/s | 4.8 MB 00:00 2024-04-02T04:47:10.058 INFO:teuthology.orchestra.run.smithi152.stdout:(7/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 2.1 MB/s | 217 kB 00:00 2024-04-02T04:47:10.073 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : cephadm-2:17.2.7-803.gc924f991.el9.noarch 1/1 2024-04-02T04:47:10.073 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:10.073 INFO:teuthology.orchestra.run.smithi154.stdout:Installed: 2024-04-02T04:47:10.073 INFO:teuthology.orchestra.run.smithi154.stdout: cephadm-2:17.2.7-803.gc924f991.el9.noarch 2024-04-02T04:47:10.073 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:10.073 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-04-02T04:47:10.234 INFO:teuthology.orchestra.run.smithi152.stdout:(8/12): libgfortran-11.4.1-3.el9.x86_64.rpm 4.5 MB/s | 803 kB 00:00 2024-04-02T04:47:10.285 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum -y install ceph-immutable-object-cache 2024-04-02T04:47:10.418 INFO:teuthology.orchestra.run.smithi152.stdout:(9/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 40 MB/s | 16 MB 00:00 2024-04-02T04:47:10.443 INFO:teuthology.orchestra.run.smithi152.stdout:(10/12): libquadmath-11.4.1-3.el9.x86_64.rpm 947 kB/s | 195 kB 00:00 2024-04-02T04:47:10.560 INFO:teuthology.orchestra.run.smithi152.stdout:(11/12): python3-devel-3.9.17-2.el9.x86_64.rpm 1.7 MB/s | 245 kB 00:00 2024-04-02T04:47:10.857 INFO:teuthology.orchestra.run.smithi154.stdout:Last metadata expiration check: 0:00:55 ago on Tue 02 Apr 2024 04:46:15 AM UTC. 2024-04-02T04:47:11.010 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-04-02T04:47:11.011 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:11.011 INFO:teuthology.orchestra.run.smithi154.stdout: Package Arch Version Repo Size 2024-04-02T04:47:11.011 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:11.011 INFO:teuthology.orchestra.run.smithi154.stdout:Installing: 2024-04-02T04:47:11.011 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-803.gc924f991.el9 ceph 137 k 2024-04-02T04:47:11.011 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:11.011 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction Summary 2024-04-02T04:47:11.011 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:11.011 INFO:teuthology.orchestra.run.smithi154.stdout:Install 1 Package 2024-04-02T04:47:11.011 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:11.012 INFO:teuthology.orchestra.run.smithi154.stdout:Total download size: 137 k 2024-04-02T04:47:11.012 INFO:teuthology.orchestra.run.smithi154.stdout:Installed size: 379 k 2024-04-02T04:47:11.012 INFO:teuthology.orchestra.run.smithi154.stdout:Downloading Packages: 2024-04-02T04:47:11.235 INFO:teuthology.orchestra.run.smithi152.stdout:(12/12): ceph-mgr-diskprediction-local-17.2.7-8 4.2 MB/s | 7.4 MB 00:01 2024-04-02T04:47:11.237 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:47:11.237 INFO:teuthology.orchestra.run.smithi152.stdout:Total 20 MB/s | 37 MB 00:01 2024-04-02T04:47:11.279 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-02T04:47:11.303 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-02T04:47:11.303 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-02T04:47:11.346 INFO:teuthology.orchestra.run.smithi154.stdout:ceph-immutable-object-cache-17.2.7-803.gc924f99 411 kB/s | 137 kB 00:00 2024-04-02T04:47:11.347 INFO:teuthology.orchestra.run.smithi154.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:47:11.347 INFO:teuthology.orchestra.run.smithi154.stdout:Total 409 kB/s | 137 kB 00:00 2024-04-02T04:47:11.347 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction check 2024-04-02T04:47:11.354 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction check succeeded. 2024-04-02T04:47:11.354 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction test 2024-04-02T04:47:11.407 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction test succeeded. 2024-04-02T04:47:11.407 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction 2024-04-02T04:47:11.549 INFO:teuthology.orchestra.run.smithi154.stdout: Preparing : 1/1 2024-04-02T04:47:11.621 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-803.gc924f991.e 1/1 2024-04-02T04:47:11.646 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-803.gc924f991.e 1/1 2024-04-02T04:47:11.650 INFO:teuthology.orchestra.run.smithi154.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-02T04:47:11.650 INFO:teuthology.orchestra.run.smithi154.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-04-02T04:47:11.650 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:11.728 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-02T04:47:11.728 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-02T04:47:12.429 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-02T04:47:12.508 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 1/12 2024-04-02T04:47:12.547 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-04-02T04:47:12.622 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-803.gc924f991.e 1/1 2024-04-02T04:47:12.622 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:12.622 INFO:teuthology.orchestra.run.smithi154.stdout:Installed: 2024-04-02T04:47:12.622 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-immutable-object-cache-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:47:12.622 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:12.622 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-04-02T04:47:12.643 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 3/12 2024-04-02T04:47:12.684 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-devel-3.9.17-2.el9.x86_64 4/12 2024-04-02T04:47:12.849 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum -y install ceph-mgr 2024-04-02T04:47:12.862 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-02T04:47:12.891 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-02T04:47:12.982 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-04-02T04:47:13.425 INFO:teuthology.orchestra.run.smithi154.stdout:Last metadata expiration check: 0:00:58 ago on Tue 02 Apr 2024 04:46:15 AM UTC. 2024-04-02T04:47:13.523 INFO:teuthology.orchestra.run.smithi154.stdout:Package ceph-mgr-2:17.2.7-803.gc924f991.el9.x86_64 is already installed. 2024-04-02T04:47:13.574 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-04-02T04:47:13.575 INFO:teuthology.orchestra.run.smithi154.stdout:Nothing to do. 2024-04-02T04:47:13.575 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-04-02T04:47:13.646 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum -y install ceph-mgr-dashboard 2024-04-02T04:47:13.647 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-04-02T04:47:13.778 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-04-02T04:47:14.205 INFO:teuthology.orchestra.run.smithi154.stdout:Last metadata expiration check: 0:00:59 ago on Tue 02 Apr 2024 04:46:15 AM UTC. 2024-04-02T04:47:14.362 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-04-02T04:47:14.363 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:14.363 INFO:teuthology.orchestra.run.smithi154.stdout: Package Arch Version Repository Size 2024-04-02T04:47:14.363 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:14.363 INFO:teuthology.orchestra.run.smithi154.stdout:Installing: 2024-04-02T04:47:14.363 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mgr-dashboard noarch 2:17.2.7-803.gc924f991.el9 ceph-noarch 1.7 M 2024-04-02T04:47:14.363 INFO:teuthology.orchestra.run.smithi154.stdout:Installing dependencies: 2024-04-02T04:47:14.363 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-grafana-dashboards noarch 2:17.2.7-803.gc924f991.el9 ceph-noarch 23 k 2024-04-02T04:47:14.364 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-prometheus-alerts noarch 2:17.2.7-803.gc924f991.el9 ceph-noarch 15 k 2024-04-02T04:47:14.364 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-04-02T04:47:14.364 INFO:teuthology.orchestra.run.smithi154.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-04-02T04:47:14.364 INFO:teuthology.orchestra.run.smithi154.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-04-02T04:47:14.364 INFO:teuthology.orchestra.run.smithi154.stdout:Installing weak dependencies: 2024-04-02T04:47:14.364 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-04-02T04:47:14.364 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:14.364 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction Summary 2024-04-02T04:47:14.364 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:14.364 INFO:teuthology.orchestra.run.smithi154.stdout:Install 7 Packages 2024-04-02T04:47:14.364 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:14.365 INFO:teuthology.orchestra.run.smithi154.stdout:Total download size: 2.0 M 2024-04-02T04:47:14.365 INFO:teuthology.orchestra.run.smithi154.stdout:Installed size: 8.1 M 2024-04-02T04:47:14.365 INFO:teuthology.orchestra.run.smithi154.stdout:Downloading Packages: 2024-04-02T04:47:14.640 INFO:teuthology.orchestra.run.smithi154.stdout:(1/7): ceph-prometheus-alerts-17.2.7-803.gc924f 76 kB/s | 15 kB 00:00 2024-04-02T04:47:14.683 INFO:teuthology.orchestra.run.smithi154.stdout:(2/7): ceph-grafana-dashboards-17.2.7-803.gc924 100 kB/s | 23 kB 00:00 2024-04-02T04:47:14.816 INFO:teuthology.orchestra.run.smithi154.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 52 kB/s | 9.0 kB 00:00 2024-04-02T04:47:14.975 INFO:teuthology.orchestra.run.smithi154.stdout:(4/7): ceph-mgr-dashboard-17.2.7-803.gc924f991. 3.2 MB/s | 1.7 MB 00:00 2024-04-02T04:47:14.992 INFO:teuthology.orchestra.run.smithi154.stdout:(5/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 175 kB/s | 31 kB 00:00 2024-04-02T04:47:15.034 INFO:teuthology.orchestra.run.smithi154.stdout:(6/7): python3-jwt-2.4.0-1.el9.noarch.rpm 116 kB/s | 41 kB 00:00 2024-04-02T04:47:15.067 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-04-02T04:47:15.234 INFO:teuthology.orchestra.run.smithi154.stdout:(7/7): python3-routes-2.5.1-5.el9.noarch.rpm 728 kB/s | 188 kB 00:00 2024-04-02T04:47:15.235 INFO:teuthology.orchestra.run.smithi154.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:47:15.236 INFO:teuthology.orchestra.run.smithi154.stdout:Total 2.3 MB/s | 2.0 MB 00:00 2024-04-02T04:47:15.236 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction check 2024-04-02T04:47:15.243 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction check succeeded. 2024-04-02T04:47:15.244 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction test 2024-04-02T04:47:15.307 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction test succeeded. 2024-04-02T04:47:15.307 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction 2024-04-02T04:47:15.599 INFO:teuthology.orchestra.run.smithi154.stdout: Preparing : 1/1 2024-04-02T04:47:15.643 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-04-02T04:47:15.669 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-04-02T04:47:15.715 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-04-02T04:47:15.747 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-04-02T04:47:15.776 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-803.gc924f991.el9.no 5/7 2024-04-02T04:47:16.003 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-803.gc924f991.el9.n 6/7 2024-04-02T04:47:16.037 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-803.gc924f991.el9.noarch 7/7 2024-04-02T04:47:16.100 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-04-02T04:47:16.122 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-803.gc924f9 12/12 2024-04-02T04:47:16.607 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-803.gc924f991.el9.noarch 7/7 2024-04-02T04:47:16.607 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-803.gc924f991.el9.n 1/7 2024-04-02T04:47:16.607 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-803.gc924f991.el9.noarch 2/7 2024-04-02T04:47:16.607 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-803.gc924f991.el9.no 3/7 2024-04-02T04:47:16.607 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-04-02T04:47:16.607 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-04-02T04:47:16.607 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-04-02T04:47:16.897 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-04-02T04:47:16.897 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:16.897 INFO:teuthology.orchestra.run.smithi154.stdout:Installed: 2024-04-02T04:47:16.897 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-grafana-dashboards-2:17.2.7-803.gc924f991.el9.noarch 2024-04-02T04:47:16.897 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mgr-dashboard-2:17.2.7-803.gc924f991.el9.noarch 2024-04-02T04:47:16.897 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-prometheus-alerts-2:17.2.7-803.gc924f991.el9.noarch 2024-04-02T04:47:16.897 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-04-02T04:47:16.897 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-04-02T04:47:16.897 INFO:teuthology.orchestra.run.smithi154.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-04-02T04:47:16.898 INFO:teuthology.orchestra.run.smithi154.stdout: python3-routes-2.5.1-5.el9.noarch 2024-04-02T04:47:16.898 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:16.898 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-04-02T04:47:17.133 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum -y install ceph-mgr-diskprediction-local 2024-04-02T04:47:17.686 INFO:teuthology.orchestra.run.smithi154.stdout:Last metadata expiration check: 0:01:02 ago on Tue 02 Apr 2024 04:46:15 AM UTC. 2024-04-02T04:47:17.839 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-04-02T04:47:17.840 INFO:teuthology.orchestra.run.smithi154.stdout:=========================================================================================== 2024-04-02T04:47:17.840 INFO:teuthology.orchestra.run.smithi154.stdout: Package Arch Version Repository Size 2024-04-02T04:47:17.841 INFO:teuthology.orchestra.run.smithi154.stdout:=========================================================================================== 2024-04-02T04:47:17.841 INFO:teuthology.orchestra.run.smithi154.stdout:Installing: 2024-04-02T04:47:17.841 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-803.gc924f991.el9 ceph-noarch 7.4 M 2024-04-02T04:47:17.841 INFO:teuthology.orchestra.run.smithi154.stdout:Installing dependencies: 2024-04-02T04:47:17.841 INFO:teuthology.orchestra.run.smithi154.stdout: flexiblas x86_64 3.0.4-8.el9 CentOS-AppStream 32 k 2024-04-02T04:47:17.841 INFO:teuthology.orchestra.run.smithi154.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 CentOS-AppStream 3.0 M 2024-04-02T04:47:17.841 INFO:teuthology.orchestra.run.smithi154.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 CentOS-AppStream 17 k 2024-04-02T04:47:17.841 INFO:teuthology.orchestra.run.smithi154.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-04-02T04:47:17.841 INFO:teuthology.orchestra.run.smithi154.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-04-02T04:47:17.841 INFO:teuthology.orchestra.run.smithi154.stdout: openblas x86_64 0.3.21-2.el9 CentOS-AppStream 34 k 2024-04-02T04:47:17.841 INFO:teuthology.orchestra.run.smithi154.stdout: openblas-openmp x86_64 0.3.21-2.el9 CentOS-AppStream 4.7 M 2024-04-02T04:47:17.841 INFO:teuthology.orchestra.run.smithi154.stdout: python3-devel x86_64 3.9.17-2.el9 appstream 245 k 2024-04-02T04:47:17.841 INFO:teuthology.orchestra.run.smithi154.stdout: python3-numpy x86_64 1:1.20.1-5.el9 CentOS-AppStream 4.8 M 2024-04-02T04:47:17.841 INFO:teuthology.orchestra.run.smithi154.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 CentOS-AppStream 217 k 2024-04-02T04:47:17.842 INFO:teuthology.orchestra.run.smithi154.stdout: python3-scipy x86_64 1.6.2-8.el9 CentOS-AppStream 16 M 2024-04-02T04:47:17.842 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:17.842 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction Summary 2024-04-02T04:47:17.842 INFO:teuthology.orchestra.run.smithi154.stdout:=========================================================================================== 2024-04-02T04:47:17.842 INFO:teuthology.orchestra.run.smithi154.stdout:Install 12 Packages 2024-04-02T04:47:17.842 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:17.842 INFO:teuthology.orchestra.run.smithi154.stdout:Total download size: 37 M 2024-04-02T04:47:17.842 INFO:teuthology.orchestra.run.smithi154.stdout:Installed size: 209 M 2024-04-02T04:47:17.843 INFO:teuthology.orchestra.run.smithi154.stdout:Downloading Packages: 2024-04-02T04:47:18.249 INFO:teuthology.orchestra.run.smithi154.stdout:(1/12): flexiblas-3.0.4-8.el9.x86_64.rpm 224 kB/s | 32 kB 00:00 2024-04-02T04:47:18.282 INFO:teuthology.orchestra.run.smithi154.stdout:(2/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 533 kB/s | 17 kB 00:00 2024-04-02T04:47:18.324 INFO:teuthology.orchestra.run.smithi154.stdout:(3/12): openblas-0.3.21-2.el9.x86_64.rpm 848 kB/s | 34 kB 00:00 2024-04-02T04:47:18.383 INFO:teuthology.orchestra.run.smithi154.stdout:(4/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 11 MB/s | 3.0 MB 00:00 2024-04-02T04:47:18.558 INFO:teuthology.orchestra.run.smithi154.stdout:(5/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 20 MB/s | 4.7 MB 00:00 2024-04-02T04:47:18.578 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-803.gc924f9 12/12 2024-04-02T04:47:18.578 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-803.gc924f9 1/12 2024-04-02T04:47:18.578 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 2/12 2024-04-02T04:47:18.578 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 3/12 2024-04-02T04:47:18.578 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 4/12 2024-04-02T04:47:18.579 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-02T04:47:18.579 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-02T04:47:18.579 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 7/12 2024-04-02T04:47:18.579 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 8/12 2024-04-02T04:47:18.579 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 9/12 2024-04-02T04:47:18.579 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 10/12 2024-04-02T04:47:18.579 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 11/12 2024-04-02T04:47:18.625 INFO:teuthology.orchestra.run.smithi154.stdout:(6/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 20 MB/s | 4.8 MB 00:00 2024-04-02T04:47:18.650 INFO:teuthology.orchestra.run.smithi154.stdout:(7/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 2.4 MB/s | 217 kB 00:00 2024-04-02T04:47:18.876 INFO:teuthology.orchestra.run.smithi154.stdout:(8/12): ceph-mgr-diskprediction-local-17.2.7-80 9.6 MB/s | 7.4 MB 00:00 2024-04-02T04:47:19.006 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-devel-3.9.17-2.el9.x86_64 12/12 2024-04-02T04:47:19.006 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:19.006 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-02T04:47:19.006 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-diskprediction-local-2:17.2.7-803.gc924f991.el9.noarch 2024-04-02T04:47:19.006 INFO:teuthology.orchestra.run.smithi152.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-04-02T04:47:19.006 INFO:teuthology.orchestra.run.smithi152.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-04-02T04:47:19.006 INFO:teuthology.orchestra.run.smithi152.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-04-02T04:47:19.007 INFO:teuthology.orchestra.run.smithi152.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-04-02T04:47:19.007 INFO:teuthology.orchestra.run.smithi152.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-04-02T04:47:19.007 INFO:teuthology.orchestra.run.smithi152.stdout: openblas-0.3.21-2.el9.x86_64 2024-04-02T04:47:19.007 INFO:teuthology.orchestra.run.smithi152.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-04-02T04:47:19.007 INFO:teuthology.orchestra.run.smithi152.stdout: python3-devel-3.9.17-2.el9.x86_64 2024-04-02T04:47:19.007 INFO:teuthology.orchestra.run.smithi152.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-04-02T04:47:19.007 INFO:teuthology.orchestra.run.smithi152.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-04-02T04:47:19.007 INFO:teuthology.orchestra.run.smithi152.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-04-02T04:47:19.007 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:19.007 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-02T04:47:19.043 INFO:teuthology.orchestra.run.smithi154.stdout:(9/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 38 MB/s | 16 MB 00:00 2024-04-02T04:47:19.068 INFO:teuthology.orchestra.run.smithi154.stdout:(10/12): libgfortran-11.4.1-3.el9.x86_64.rpm 1.9 MB/s | 803 kB 00:00 2024-04-02T04:47:19.152 INFO:teuthology.orchestra.run.smithi154.stdout:(11/12): libquadmath-11.4.1-3.el9.x86_64.rpm 714 kB/s | 195 kB 00:00 2024-04-02T04:47:19.377 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-mgr-rook 2024-04-02T04:47:19.961 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:13 ago on Tue 02 Apr 2024 04:46:06 AM UTC. 2024-04-02T04:47:20.118 INFO:teuthology.orchestra.run.smithi154.stdout:(12/12): python3-devel-3.9.17-2.el9.x86_64.rpm 228 kB/s | 245 kB 00:01 2024-04-02T04:47:20.121 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-02T04:47:20.121 INFO:teuthology.orchestra.run.smithi154.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:47:20.122 INFO:teuthology.orchestra.run.smithi154.stdout:Total 16 MB/s | 37 MB 00:02 2024-04-02T04:47:20.122 INFO:teuthology.orchestra.run.smithi152.stdout:======================================================================================= 2024-04-02T04:47:20.122 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-02T04:47:20.123 INFO:teuthology.orchestra.run.smithi152.stdout:======================================================================================= 2024-04-02T04:47:20.123 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-02T04:47:20.123 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-rook noarch 2:17.2.7-803.gc924f991.el9 ceph-noarch 47 k 2024-04-02T04:47:20.123 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-02T04:47:20.123 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-04-02T04:47:20.123 INFO:teuthology.orchestra.run.smithi152.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-04-02T04:47:20.123 INFO:teuthology.orchestra.run.smithi152.stdout: python3-google-auth noarch 1:2.28.2-1.el9 epel 203 k 2024-04-02T04:47:20.123 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jsonpatch noarch 1.21-16.el9 CentOS-AppStream 24 k 2024-04-02T04:47:20.123 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jsonpointer noarch 2.0-4.el9 CentOS-AppStream 18 k 2024-04-02T04:47:20.123 INFO:teuthology.orchestra.run.smithi152.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-04-02T04:47:20.123 INFO:teuthology.orchestra.run.smithi152.stdout: python3-oauthlib noarch 3.1.1-5.el9 CentOS-AppStream 174 k 2024-04-02T04:47:20.123 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyasn1 noarch 0.4.8-6.el9 CentOS-AppStream 132 k 2024-04-02T04:47:20.123 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 CentOS-AppStream 211 k 2024-04-02T04:47:20.123 INFO:teuthology.orchestra.run.smithi152.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 CentOS-AppStream 43 k 2024-04-02T04:47:20.123 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-04-02T04:47:20.123 INFO:teuthology.orchestra.run.smithi152.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-04-02T04:47:20.124 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:20.124 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-02T04:47:20.124 INFO:teuthology.orchestra.run.smithi152.stdout:======================================================================================= 2024-04-02T04:47:20.124 INFO:teuthology.orchestra.run.smithi152.stdout:Install 13 Packages 2024-04-02T04:47:20.124 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:20.124 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 2.0 M 2024-04-02T04:47:20.124 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 26 M 2024-04-02T04:47:20.124 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-02T04:47:20.166 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction check 2024-04-02T04:47:20.192 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction check succeeded. 2024-04-02T04:47:20.192 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction test 2024-04-02T04:47:20.338 INFO:teuthology.orchestra.run.smithi152.stdout:(1/13): python3-jsonpatch-1.21-16.el9.noarch.rp 188 kB/s | 24 kB 00:00 2024-04-02T04:47:20.364 INFO:teuthology.orchestra.run.smithi152.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 117 kB/s | 18 kB 00:00 2024-04-02T04:47:20.414 INFO:teuthology.orchestra.run.smithi152.stdout:(3/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 2.3 MB/s | 174 kB 00:00 2024-04-02T04:47:20.439 INFO:teuthology.orchestra.run.smithi152.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.7 MB/s | 132 kB 00:00 2024-04-02T04:47:20.464 INFO:teuthology.orchestra.run.smithi152.stdout:(5/13): python3-pyasn1-modules-0.4.8-6.el9.noar 4.1 MB/s | 211 kB 00:00 2024-04-02T04:47:20.490 INFO:teuthology.orchestra.run.smithi152.stdout:(6/13): ceph-mgr-rook-17.2.7-803.gc924f991.el9. 169 kB/s | 47 kB 00:00 2024-04-02T04:47:20.515 INFO:teuthology.orchestra.run.smithi152.stdout:(7/13): python3-requests-oauthlib-1.3.0-12.el9. 569 kB/s | 43 kB 00:00 2024-04-02T04:47:20.644 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction test succeeded. 2024-04-02T04:47:20.644 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction 2024-04-02T04:47:20.740 INFO:teuthology.orchestra.run.smithi152.stdout:(8/13): python3-certifi-2021.10.8-2.el9.noarch. 67 kB/s | 15 kB 00:00 2024-04-02T04:47:20.799 INFO:teuthology.orchestra.run.smithi152.stdout:(9/13): python3-cachetools-4.2.4-1.el9.noarch.r 96 kB/s | 32 kB 00:00 2024-04-02T04:47:20.907 INFO:teuthology.orchestra.run.smithi152.stdout:(10/13): python3-google-auth-2.28.2-1.el9.noarc 517 kB/s | 203 kB 00:00 2024-04-02T04:47:21.258 INFO:teuthology.orchestra.run.smithi152.stdout:(11/13): python3-kubernetes-26.1.0-2.el9.noarch 2.0 MB/s | 1.0 MB 00:00 2024-04-02T04:47:21.283 INFO:teuthology.orchestra.run.smithi152.stdout:(12/13): python3-rsa-4.9-2.el9.noarch.rpm 122 kB/s | 59 kB 00:00 2024-04-02T04:47:21.324 INFO:teuthology.orchestra.run.smithi152.stdout:(13/13): python3-websocket-client-1.2.3-2.el9.n 215 kB/s | 90 kB 00:00 2024-04-02T04:47:21.329 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:47:21.329 INFO:teuthology.orchestra.run.smithi152.stdout:Total 1.7 MB/s | 2.0 MB 00:01 2024-04-02T04:47:21.329 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-02T04:47:21.342 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-02T04:47:21.342 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-02T04:47:21.358 INFO:teuthology.orchestra.run.smithi154.stdout: Preparing : 1/1 2024-04-02T04:47:21.437 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-02T04:47:21.439 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-02T04:47:21.449 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 1/12 2024-04-02T04:47:21.481 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-04-02T04:47:21.566 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 3/12 2024-04-02T04:47:21.609 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-devel-3.9.17-2.el9.x86_64 4/12 2024-04-02T04:47:21.787 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-02T04:47:21.817 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-02T04:47:21.927 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-04-02T04:47:21.958 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-02T04:47:22.133 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-04-02T04:47:22.192 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-04-02T04:47:22.248 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-04-02T04:47:22.285 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-04-02T04:47:22.326 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-04-02T04:47:22.429 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-04-02T04:47:22.545 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-google-auth-1:2.28.2-1.el9.noarch 7/13 2024-04-02T04:47:22.594 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-04-02T04:47:22.595 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-04-02T04:47:22.696 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-04-02T04:47:23.332 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-04-02T04:47:23.382 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-04-02T04:47:23.416 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-04-02T04:47:23.462 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-04-02T04:47:23.486 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-rook-2:17.2.7-803.gc924f991.el9.noarch 13/13 2024-04-02T04:47:23.944 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-04-02T04:47:24.285 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-803.gc924f991.el9.noarch 13/13 2024-04-02T04:47:24.285 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-rook-2:17.2.7-803.gc924f991.el9.noarch 1/13 2024-04-02T04:47:24.285 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-04-02T04:47:24.285 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-04-02T04:47:24.285 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-04-02T04:47:24.285 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-04-02T04:47:24.285 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-04-02T04:47:24.285 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-04-02T04:47:24.285 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-04-02T04:47:24.285 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-04-02T04:47:24.285 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-google-auth-1:2.28.2-1.el9.noarch 10/13 2024-04-02T04:47:24.286 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-04-02T04:47:24.286 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-04-02T04:47:24.644 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-04-02T04:47:24.644 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:24.644 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-02T04:47:24.644 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-rook-2:17.2.7-803.gc924f991.el9.noarch 2024-04-02T04:47:24.644 INFO:teuthology.orchestra.run.smithi152.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-04-02T04:47:24.644 INFO:teuthology.orchestra.run.smithi152.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-04-02T04:47:24.644 INFO:teuthology.orchestra.run.smithi152.stdout: python3-google-auth-1:2.28.2-1.el9.noarch 2024-04-02T04:47:24.644 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-04-02T04:47:24.644 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-04-02T04:47:24.644 INFO:teuthology.orchestra.run.smithi152.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-04-02T04:47:24.644 INFO:teuthology.orchestra.run.smithi152.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-04-02T04:47:24.644 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-04-02T04:47:24.644 INFO:teuthology.orchestra.run.smithi152.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-04-02T04:47:24.645 INFO:teuthology.orchestra.run.smithi152.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-04-02T04:47:24.645 INFO:teuthology.orchestra.run.smithi152.stdout: python3-rsa-4.9-2.el9.noarch 2024-04-02T04:47:24.645 INFO:teuthology.orchestra.run.smithi152.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-04-02T04:47:24.645 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:24.645 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-02T04:47:24.869 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-mgr-cephadm 2024-04-02T04:47:24.971 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-04-02T04:47:24.998 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-803.gc924f9 12/12 2024-04-02T04:47:25.439 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:19 ago on Tue 02 Apr 2024 04:46:06 AM UTC. 2024-04-02T04:47:25.588 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-02T04:47:25.589 INFO:teuthology.orchestra.run.smithi152.stdout:======================================================================================= 2024-04-02T04:47:25.589 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-02T04:47:25.589 INFO:teuthology.orchestra.run.smithi152.stdout:======================================================================================= 2024-04-02T04:47:25.589 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-02T04:47:25.589 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-cephadm noarch 2:17.2.7-803.gc924f991.el9 ceph-noarch 123 k 2024-04-02T04:47:25.589 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-02T04:47:25.590 INFO:teuthology.orchestra.run.smithi152.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-04-02T04:47:25.590 INFO:teuthology.orchestra.run.smithi152.stdout: python3-babel noarch 2.9.1-2.el9 CentOS-AppStream 5.8 M 2024-04-02T04:47:25.590 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-04-02T04:47:25.590 INFO:teuthology.orchestra.run.smithi152.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-04-02T04:47:25.590 INFO:teuthology.orchestra.run.smithi152.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-04-02T04:47:25.590 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:25.590 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-02T04:47:25.590 INFO:teuthology.orchestra.run.smithi152.stdout:======================================================================================= 2024-04-02T04:47:25.590 INFO:teuthology.orchestra.run.smithi152.stdout:Install 6 Packages 2024-04-02T04:47:25.590 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:25.591 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 6.8 M 2024-04-02T04:47:25.591 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 33 M 2024-04-02T04:47:25.591 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-02T04:47:26.041 INFO:teuthology.orchestra.run.smithi152.stdout:(1/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 1.3 MB/s | 248 kB 00:00 2024-04-02T04:47:26.133 INFO:teuthology.orchestra.run.smithi152.stdout:(2/6): python3-babel-2.9.1-2.el9.noarch.rpm 21 MB/s | 5.8 MB 00:00 2024-04-02T04:47:26.184 INFO:teuthology.orchestra.run.smithi152.stdout:(3/6): ceph-mgr-cephadm-17.2.7-803.gc924f991.el 376 kB/s | 123 kB 00:00 2024-04-02T04:47:26.367 INFO:teuthology.orchestra.run.smithi152.stdout:(4/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 1.7 MB/s | 548 kB 00:00 2024-04-02T04:47:26.392 INFO:teuthology.orchestra.run.smithi152.stdout:(5/6): python3-natsort-7.1.1-5.el9.noarch.rpm 245 kB/s | 58 kB 00:00 2024-04-02T04:47:26.417 INFO:teuthology.orchestra.run.smithi152.stdout:(6/6): python3-typing-extensions-4.4.0-2.el9.no 244 kB/s | 51 kB 00:00 2024-04-02T04:47:26.420 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:47:26.420 INFO:teuthology.orchestra.run.smithi152.stdout:Total 8.2 MB/s | 6.8 MB 00:00 2024-04-02T04:47:26.434 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-02T04:47:26.445 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-02T04:47:26.445 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-02T04:47:26.533 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-02T04:47:26.534 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-02T04:47:26.835 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-02T04:47:27.003 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-04-02T04:47:27.070 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-04-02T04:47:27.402 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-803.gc924f9 12/12 2024-04-02T04:47:27.402 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-803.gc924f9 1/12 2024-04-02T04:47:27.402 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 2/12 2024-04-02T04:47:27.402 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 3/12 2024-04-02T04:47:27.402 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 4/12 2024-04-02T04:47:27.402 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 5/12 2024-04-02T04:47:27.402 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-04-02T04:47:27.402 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 7/12 2024-04-02T04:47:27.402 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 8/12 2024-04-02T04:47:27.403 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 9/12 2024-04-02T04:47:27.403 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 10/12 2024-04-02T04:47:27.403 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 11/12 2024-04-02T04:47:27.607 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-04-02T04:47:27.699 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-04-02T04:47:27.754 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-04-02T04:47:27.783 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-803.gc924f991.el9.noarch 6/6 2024-04-02T04:47:27.815 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-devel-3.9.17-2.el9.x86_64 12/12 2024-04-02T04:47:27.815 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:27.815 INFO:teuthology.orchestra.run.smithi154.stdout:Installed: 2024-04-02T04:47:27.815 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mgr-diskprediction-local-2:17.2.7-803.gc924f991.el9.noarch 2024-04-02T04:47:27.815 INFO:teuthology.orchestra.run.smithi154.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-04-02T04:47:27.815 INFO:teuthology.orchestra.run.smithi154.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-04-02T04:47:27.815 INFO:teuthology.orchestra.run.smithi154.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-04-02T04:47:27.815 INFO:teuthology.orchestra.run.smithi154.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-04-02T04:47:27.815 INFO:teuthology.orchestra.run.smithi154.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-04-02T04:47:27.815 INFO:teuthology.orchestra.run.smithi154.stdout: openblas-0.3.21-2.el9.x86_64 2024-04-02T04:47:27.815 INFO:teuthology.orchestra.run.smithi154.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-04-02T04:47:27.816 INFO:teuthology.orchestra.run.smithi154.stdout: python3-devel-3.9.17-2.el9.x86_64 2024-04-02T04:47:27.816 INFO:teuthology.orchestra.run.smithi154.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-04-02T04:47:27.816 INFO:teuthology.orchestra.run.smithi154.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-04-02T04:47:27.816 INFO:teuthology.orchestra.run.smithi154.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-04-02T04:47:27.816 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:27.816 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-04-02T04:47:28.191 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum -y install ceph-mgr-rook 2024-04-02T04:47:28.529 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-803.gc924f991.el9.noarch 6/6 2024-04-02T04:47:28.529 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-803.gc924f991.el9.noarch 1/6 2024-04-02T04:47:28.529 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-04-02T04:47:28.529 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-04-02T04:47:28.529 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-04-02T04:47:28.529 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-04-02T04:47:28.763 INFO:teuthology.orchestra.run.smithi154.stdout:Last metadata expiration check: 0:01:13 ago on Tue 02 Apr 2024 04:46:15 AM UTC. 2024-04-02T04:47:28.825 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-04-02T04:47:28.825 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:28.825 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-02T04:47:28.825 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-mgr-cephadm-2:17.2.7-803.gc924f991.el9.noarch 2024-04-02T04:47:28.826 INFO:teuthology.orchestra.run.smithi152.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-04-02T04:47:28.826 INFO:teuthology.orchestra.run.smithi152.stdout: python3-babel-2.9.1-2.el9.noarch 2024-04-02T04:47:28.826 INFO:teuthology.orchestra.run.smithi152.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-04-02T04:47:28.826 INFO:teuthology.orchestra.run.smithi152.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-04-02T04:47:28.826 INFO:teuthology.orchestra.run.smithi152.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-04-02T04:47:28.826 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:28.826 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-02T04:47:28.915 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-04-02T04:47:28.916 INFO:teuthology.orchestra.run.smithi154.stdout:======================================================================================= 2024-04-02T04:47:28.916 INFO:teuthology.orchestra.run.smithi154.stdout: Package Arch Version Repository Size 2024-04-02T04:47:28.916 INFO:teuthology.orchestra.run.smithi154.stdout:======================================================================================= 2024-04-02T04:47:28.916 INFO:teuthology.orchestra.run.smithi154.stdout:Installing: 2024-04-02T04:47:28.916 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mgr-rook noarch 2:17.2.7-803.gc924f991.el9 ceph-noarch 47 k 2024-04-02T04:47:28.916 INFO:teuthology.orchestra.run.smithi154.stdout:Installing dependencies: 2024-04-02T04:47:28.916 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-04-02T04:47:28.916 INFO:teuthology.orchestra.run.smithi154.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-04-02T04:47:28.916 INFO:teuthology.orchestra.run.smithi154.stdout: python3-google-auth noarch 1:2.28.2-1.el9 epel 203 k 2024-04-02T04:47:28.916 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jsonpatch noarch 1.21-16.el9 CentOS-AppStream 24 k 2024-04-02T04:47:28.917 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jsonpointer noarch 2.0-4.el9 CentOS-AppStream 18 k 2024-04-02T04:47:28.917 INFO:teuthology.orchestra.run.smithi154.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-04-02T04:47:28.917 INFO:teuthology.orchestra.run.smithi154.stdout: python3-oauthlib noarch 3.1.1-5.el9 CentOS-AppStream 174 k 2024-04-02T04:47:28.917 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pyasn1 noarch 0.4.8-6.el9 CentOS-AppStream 132 k 2024-04-02T04:47:28.917 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 CentOS-AppStream 211 k 2024-04-02T04:47:28.917 INFO:teuthology.orchestra.run.smithi154.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 CentOS-AppStream 43 k 2024-04-02T04:47:28.917 INFO:teuthology.orchestra.run.smithi154.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-04-02T04:47:28.917 INFO:teuthology.orchestra.run.smithi154.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-04-02T04:47:28.917 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:28.917 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction Summary 2024-04-02T04:47:28.917 INFO:teuthology.orchestra.run.smithi154.stdout:======================================================================================= 2024-04-02T04:47:28.917 INFO:teuthology.orchestra.run.smithi154.stdout:Install 13 Packages 2024-04-02T04:47:28.917 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:28.918 INFO:teuthology.orchestra.run.smithi154.stdout:Total download size: 2.0 M 2024-04-02T04:47:28.918 INFO:teuthology.orchestra.run.smithi154.stdout:Installed size: 26 M 2024-04-02T04:47:28.918 INFO:teuthology.orchestra.run.smithi154.stdout:Downloading Packages: 2024-04-02T04:47:29.078 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install ceph-fuse 2024-04-02T04:47:29.206 INFO:teuthology.orchestra.run.smithi154.stdout:(1/13): python3-jsonpointer-2.0-4.el9.noarch.rp 148 kB/s | 18 kB 00:00 2024-04-02T04:47:29.231 INFO:teuthology.orchestra.run.smithi154.stdout:(2/13): python3-jsonpatch-1.21-16.el9.noarch.rp 165 kB/s | 24 kB 00:00 2024-04-02T04:47:29.282 INFO:teuthology.orchestra.run.smithi154.stdout:(3/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 2.3 MB/s | 174 kB 00:00 2024-04-02T04:47:29.307 INFO:teuthology.orchestra.run.smithi154.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.7 MB/s | 132 kB 00:00 2024-04-02T04:47:29.332 INFO:teuthology.orchestra.run.smithi154.stdout:(5/13): python3-pyasn1-modules-0.4.8-6.el9.noar 4.1 MB/s | 211 kB 00:00 2024-04-02T04:47:29.358 INFO:teuthology.orchestra.run.smithi154.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 852 kB/s | 43 kB 00:00 2024-04-02T04:47:29.383 INFO:teuthology.orchestra.run.smithi154.stdout:(7/13): ceph-mgr-rook-17.2.7-803.gc924f991.el9. 159 kB/s | 47 kB 00:00 2024-04-02T04:47:29.642 INFO:teuthology.orchestra.run.smithi154.stdout:(8/13): python3-certifi-2021.10.8-2.el9.noarch. 53 kB/s | 15 kB 00:00 2024-04-02T04:47:29.643 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:23 ago on Tue 02 Apr 2024 04:46:06 AM UTC. 2024-04-02T04:47:29.667 INFO:teuthology.orchestra.run.smithi154.stdout:(9/13): python3-cachetools-4.2.4-1.el9.noarch.r 96 kB/s | 32 kB 00:00 2024-04-02T04:47:29.794 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-02T04:47:29.795 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:29.795 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repository Size 2024-04-02T04:47:29.795 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:29.795 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-02T04:47:29.795 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-fuse x86_64 2:17.2.7-803.gc924f991.el9 ceph 813 k 2024-04-02T04:47:29.795 INFO:teuthology.orchestra.run.smithi152.stdout:Installing dependencies: 2024-04-02T04:47:29.795 INFO:teuthology.orchestra.run.smithi152.stdout: fuse x86_64 2.9.9-15.el9 CentOS-BaseOS 79 k 2024-04-02T04:47:29.795 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:29.795 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-02T04:47:29.795 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:29.796 INFO:teuthology.orchestra.run.smithi152.stdout:Install 2 Packages 2024-04-02T04:47:29.796 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:29.796 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 891 k 2024-04-02T04:47:29.796 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 2.6 M 2024-04-02T04:47:29.796 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-02T04:47:29.809 INFO:teuthology.orchestra.run.smithi154.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 417 kB/s | 59 kB 00:00 2024-04-02T04:47:29.834 INFO:teuthology.orchestra.run.smithi154.stdout:(11/13): python3-google-auth-2.28.2-1.el9.noarc 450 kB/s | 203 kB 00:00 2024-04-02T04:47:29.920 INFO:teuthology.orchestra.run.smithi152.stdout:(1/2): fuse-2.9.9-15.el9.x86_64.rpm 635 kB/s | 79 kB 00:00 2024-04-02T04:47:29.926 INFO:teuthology.orchestra.run.smithi154.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 769 kB/s | 90 kB 00:00 2024-04-02T04:47:30.018 INFO:teuthology.orchestra.run.smithi154.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 2.7 MB/s | 1.0 MB 00:00 2024-04-02T04:47:30.021 INFO:teuthology.orchestra.run.smithi154.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:47:30.022 INFO:teuthology.orchestra.run.smithi154.stdout:Total 1.9 MB/s | 2.0 MB 00:01 2024-04-02T04:47:30.022 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction check 2024-04-02T04:47:30.035 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction check succeeded. 2024-04-02T04:47:30.035 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction test 2024-04-02T04:47:30.120 INFO:teuthology.orchestra.run.smithi152.stdout:(2/2): ceph-fuse-17.2.7-803.gc924f991.el9.x86_6 2.5 MB/s | 813 kB 00:00 2024-04-02T04:47:30.122 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:47:30.122 INFO:teuthology.orchestra.run.smithi152.stdout:Total 2.7 MB/s | 891 kB 00:00 2024-04-02T04:47:30.122 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-02T04:47:30.128 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction test succeeded. 2024-04-02T04:47:30.129 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction 2024-04-02T04:47:30.142 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-02T04:47:30.142 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-02T04:47:30.216 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-02T04:47:30.216 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-02T04:47:30.396 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-02T04:47:30.541 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-04-02T04:47:30.611 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : ceph-fuse-2:17.2.7-803.gc924f991.el9.x86_64 2/2 2024-04-02T04:47:30.636 INFO:teuthology.orchestra.run.smithi154.stdout: Preparing : 1/1 2024-04-02T04:47:30.806 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-04-02T04:47:30.861 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-04-02T04:47:30.958 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-04-02T04:47:30.961 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-04-02T04:47:31.002 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-04-02T04:47:31.097 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-04-02T04:47:31.197 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-google-auth-1:2.28.2-1.el9.noarch 7/13 2024-04-02T04:47:31.246 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-04-02T04:47:31.365 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: ceph-fuse-2:17.2.7-803.gc924f991.el9.x86_64 2/2 2024-04-02T04:47:31.365 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : ceph-fuse-2:17.2.7-803.gc924f991.el9.x86_64 1/2 2024-04-02T04:47:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-04-02T04:47:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:31.657 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-02T04:47:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: ceph-fuse-2:17.2.7-803.gc924f991.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-04-02T04:47:31.657 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:31.658 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-02T04:47:31.932 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install librados-devel 2024-04-02T04:47:31.969 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-04-02T04:47:32.018 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-04-02T04:47:32.052 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-04-02T04:47:32.106 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-04-02T04:47:32.130 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-mgr-rook-2:17.2.7-803.gc924f991.el9.noarch 13/13 2024-04-02T04:47:32.511 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:26 ago on Tue 02 Apr 2024 04:46:06 AM UTC. 2024-04-02T04:47:32.662 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-02T04:47:32.663 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:32.663 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repo Size 2024-04-02T04:47:32.663 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:32.663 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-02T04:47:32.663 INFO:teuthology.orchestra.run.smithi152.stdout: librados-devel x86_64 2:17.2.7-803.gc924f991.el9 ceph 126 k 2024-04-02T04:47:32.663 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:32.663 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-02T04:47:32.663 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:32.663 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-04-02T04:47:32.663 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:32.664 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 126 k 2024-04-02T04:47:32.664 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 448 k 2024-04-02T04:47:32.664 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-02T04:47:32.887 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-803.gc924f991.el9.noarch 13/13 2024-04-02T04:47:32.887 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-mgr-rook-2:17.2.7-803.gc924f991.el9.noarch 1/13 2024-04-02T04:47:32.887 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-04-02T04:47:32.887 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-04-02T04:47:32.887 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-04-02T04:47:32.887 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-04-02T04:47:32.887 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-04-02T04:47:32.887 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-04-02T04:47:32.887 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-04-02T04:47:32.887 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-04-02T04:47:32.887 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-google-auth-1:2.28.2-1.el9.noarch 10/13 2024-04-02T04:47:32.887 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-04-02T04:47:32.887 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-04-02T04:47:33.024 INFO:teuthology.orchestra.run.smithi152.stdout:librados-devel-17.2.7-803.gc924f991.el9.x86_64. 351 kB/s | 126 kB 00:00 2024-04-02T04:47:33.025 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:47:33.025 INFO:teuthology.orchestra.run.smithi152.stdout:Total 349 kB/s | 126 kB 00:00 2024-04-02T04:47:33.025 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-02T04:47:33.030 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-02T04:47:33.031 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-02T04:47:33.085 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-02T04:47:33.086 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-02T04:47:33.229 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-02T04:47:33.238 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-04-02T04:47:33.238 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:33.238 INFO:teuthology.orchestra.run.smithi154.stdout:Installed: 2024-04-02T04:47:33.238 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mgr-rook-2:17.2.7-803.gc924f991.el9.noarch 2024-04-02T04:47:33.238 INFO:teuthology.orchestra.run.smithi154.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-04-02T04:47:33.238 INFO:teuthology.orchestra.run.smithi154.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-04-02T04:47:33.238 INFO:teuthology.orchestra.run.smithi154.stdout: python3-google-auth-1:2.28.2-1.el9.noarch 2024-04-02T04:47:33.238 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-04-02T04:47:33.238 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-04-02T04:47:33.239 INFO:teuthology.orchestra.run.smithi154.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-04-02T04:47:33.239 INFO:teuthology.orchestra.run.smithi154.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-04-02T04:47:33.239 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-04-02T04:47:33.239 INFO:teuthology.orchestra.run.smithi154.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-04-02T04:47:33.239 INFO:teuthology.orchestra.run.smithi154.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-04-02T04:47:33.239 INFO:teuthology.orchestra.run.smithi154.stdout: python3-rsa-4.9-2.el9.noarch 2024-04-02T04:47:33.239 INFO:teuthology.orchestra.run.smithi154.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-04-02T04:47:33.239 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:33.239 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-04-02T04:47:33.339 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : librados-devel-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:33.483 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum -y install ceph-mgr-cephadm 2024-04-02T04:47:33.787 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: librados-devel-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:34.071 INFO:teuthology.orchestra.run.smithi154.stdout:Last metadata expiration check: 0:01:19 ago on Tue 02 Apr 2024 04:46:15 AM UTC. 2024-04-02T04:47:34.071 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : librados-devel-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:34.071 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:34.072 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-02T04:47:34.072 INFO:teuthology.orchestra.run.smithi152.stdout: librados-devel-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:47:34.072 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:34.072 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-02T04:47:34.224 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-04-02T04:47:34.225 INFO:teuthology.orchestra.run.smithi154.stdout:======================================================================================= 2024-04-02T04:47:34.225 INFO:teuthology.orchestra.run.smithi154.stdout: Package Arch Version Repository Size 2024-04-02T04:47:34.225 INFO:teuthology.orchestra.run.smithi154.stdout:======================================================================================= 2024-04-02T04:47:34.225 INFO:teuthology.orchestra.run.smithi154.stdout:Installing: 2024-04-02T04:47:34.225 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mgr-cephadm noarch 2:17.2.7-803.gc924f991.el9 ceph-noarch 123 k 2024-04-02T04:47:34.225 INFO:teuthology.orchestra.run.smithi154.stdout:Installing dependencies: 2024-04-02T04:47:34.226 INFO:teuthology.orchestra.run.smithi154.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-04-02T04:47:34.226 INFO:teuthology.orchestra.run.smithi154.stdout: python3-babel noarch 2.9.1-2.el9 CentOS-AppStream 5.8 M 2024-04-02T04:47:34.226 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-04-02T04:47:34.226 INFO:teuthology.orchestra.run.smithi154.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-04-02T04:47:34.226 INFO:teuthology.orchestra.run.smithi154.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-04-02T04:47:34.226 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:34.226 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction Summary 2024-04-02T04:47:34.226 INFO:teuthology.orchestra.run.smithi154.stdout:======================================================================================= 2024-04-02T04:47:34.226 INFO:teuthology.orchestra.run.smithi154.stdout:Install 6 Packages 2024-04-02T04:47:34.226 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:34.227 INFO:teuthology.orchestra.run.smithi154.stdout:Total download size: 6.8 M 2024-04-02T04:47:34.227 INFO:teuthology.orchestra.run.smithi154.stdout:Installed size: 33 M 2024-04-02T04:47:34.227 INFO:teuthology.orchestra.run.smithi154.stdout:Downloading Packages: 2024-04-02T04:47:34.313 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install libcephfs2 2024-04-02T04:47:34.620 INFO:teuthology.orchestra.run.smithi154.stdout:(1/6): python3-babel-2.9.1-2.el9.noarch.rpm 21 MB/s | 5.8 MB 00:00 2024-04-02T04:47:34.645 INFO:teuthology.orchestra.run.smithi154.stdout:(2/6): ceph-mgr-cephadm-17.2.7-803.gc924f991.el 408 kB/s | 123 kB 00:00 2024-04-02T04:47:34.820 INFO:teuthology.orchestra.run.smithi154.stdout:(3/6): python3-natsort-7.1.1-5.el9.noarch.rpm 330 kB/s | 58 kB 00:00 2024-04-02T04:47:34.845 INFO:teuthology.orchestra.run.smithi154.stdout:(4/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 493 kB/s | 248 kB 00:00 2024-04-02T04:47:34.870 INFO:teuthology.orchestra.run.smithi154.stdout:(5/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 2.1 MB/s | 548 kB 00:00 2024-04-02T04:47:34.895 INFO:teuthology.orchestra.run.smithi154.stdout:(6/6): python3-typing-extensions-4.4.0-2.el9.no 680 kB/s | 51 kB 00:00 2024-04-02T04:47:34.898 INFO:teuthology.orchestra.run.smithi154.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:47:34.899 INFO:teuthology.orchestra.run.smithi154.stdout:Total 10 MB/s | 6.8 MB 00:00 2024-04-02T04:47:34.901 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:28 ago on Tue 02 Apr 2024 04:46:06 AM UTC. 2024-04-02T04:47:34.911 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction check 2024-04-02T04:47:34.922 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction check succeeded. 2024-04-02T04:47:34.922 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction test 2024-04-02T04:47:35.000 INFO:teuthology.orchestra.run.smithi152.stdout:Package libcephfs2-2:17.2.7-803.gc924f991.el9.x86_64 is already installed. 2024-04-02T04:47:35.011 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction test succeeded. 2024-04-02T04:47:35.011 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction 2024-04-02T04:47:35.055 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-02T04:47:35.056 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-02T04:47:35.056 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-02T04:47:35.130 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install libcephfs-devel 2024-04-02T04:47:35.338 INFO:teuthology.orchestra.run.smithi154.stdout: Preparing : 1/1 2024-04-02T04:47:35.505 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-04-02T04:47:35.556 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-04-02T04:47:35.698 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:29 ago on Tue 02 Apr 2024 04:46:06 AM UTC. 2024-04-02T04:47:35.850 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-02T04:47:35.851 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:35.851 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repo Size 2024-04-02T04:47:35.851 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:35.851 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-02T04:47:35.851 INFO:teuthology.orchestra.run.smithi152.stdout: libcephfs-devel x86_64 2:17.2.7-803.gc924f991.el9 ceph 25 k 2024-04-02T04:47:35.851 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:35.851 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-02T04:47:35.851 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:35.851 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-04-02T04:47:35.851 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:35.852 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 25 k 2024-04-02T04:47:35.852 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 110 k 2024-04-02T04:47:35.852 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-02T04:47:36.069 INFO:teuthology.orchestra.run.smithi152.stdout:libcephfs-devel-17.2.7-803.gc924f991.el9.x86_64 115 kB/s | 25 kB 00:00 2024-04-02T04:47:36.069 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:47:36.070 INFO:teuthology.orchestra.run.smithi152.stdout:Total 114 kB/s | 25 kB 00:00 2024-04-02T04:47:36.070 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-02T04:47:36.073 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-02T04:47:36.073 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-02T04:47:36.075 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-02T04:47:36.075 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-02T04:47:36.077 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-04-02T04:47:36.159 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-04-02T04:47:36.164 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-02T04:47:36.213 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-04-02T04:47:36.243 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-803.gc924f991.el9.noarch 6/6 2024-04-02T04:47:36.275 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : libcephfs-devel-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:36.648 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:36.899 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : libcephfs-devel-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:36.899 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:36.899 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-02T04:47:36.899 INFO:teuthology.orchestra.run.smithi152.stdout: libcephfs-devel-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:47:36.899 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:36.899 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-02T04:47:37.056 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-803.gc924f991.el9.noarch 6/6 2024-04-02T04:47:37.056 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-803.gc924f991.el9.noarch 1/6 2024-04-02T04:47:37.056 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-04-02T04:47:37.056 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-04-02T04:47:37.056 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-04-02T04:47:37.056 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-04-02T04:47:37.096 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install librados2 2024-04-02T04:47:37.360 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-04-02T04:47:37.360 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:37.360 INFO:teuthology.orchestra.run.smithi154.stdout:Installed: 2024-04-02T04:47:37.360 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-mgr-cephadm-2:17.2.7-803.gc924f991.el9.noarch 2024-04-02T04:47:37.360 INFO:teuthology.orchestra.run.smithi154.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-04-02T04:47:37.360 INFO:teuthology.orchestra.run.smithi154.stdout: python3-babel-2.9.1-2.el9.noarch 2024-04-02T04:47:37.360 INFO:teuthology.orchestra.run.smithi154.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-04-02T04:47:37.361 INFO:teuthology.orchestra.run.smithi154.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-04-02T04:47:37.361 INFO:teuthology.orchestra.run.smithi154.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-04-02T04:47:37.361 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:37.361 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-04-02T04:47:37.628 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum -y install ceph-fuse 2024-04-02T04:47:37.658 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:31 ago on Tue 02 Apr 2024 04:46:06 AM UTC. 2024-04-02T04:47:37.755 INFO:teuthology.orchestra.run.smithi152.stdout:Package librados2-2:17.2.7-803.gc924f991.el9.x86_64 is already installed. 2024-04-02T04:47:37.808 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-02T04:47:37.809 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-02T04:47:37.809 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-02T04:47:37.875 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install librbd1 2024-04-02T04:47:38.189 INFO:teuthology.orchestra.run.smithi154.stdout:Last metadata expiration check: 0:01:23 ago on Tue 02 Apr 2024 04:46:15 AM UTC. 2024-04-02T04:47:38.339 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-04-02T04:47:38.340 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:38.340 INFO:teuthology.orchestra.run.smithi154.stdout: Package Arch Version Repository Size 2024-04-02T04:47:38.340 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:38.340 INFO:teuthology.orchestra.run.smithi154.stdout:Installing: 2024-04-02T04:47:38.340 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-fuse x86_64 2:17.2.7-803.gc924f991.el9 ceph 813 k 2024-04-02T04:47:38.340 INFO:teuthology.orchestra.run.smithi154.stdout:Installing dependencies: 2024-04-02T04:47:38.340 INFO:teuthology.orchestra.run.smithi154.stdout: fuse x86_64 2.9.9-15.el9 CentOS-BaseOS 79 k 2024-04-02T04:47:38.340 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:38.340 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction Summary 2024-04-02T04:47:38.340 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:38.340 INFO:teuthology.orchestra.run.smithi154.stdout:Install 2 Packages 2024-04-02T04:47:38.340 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:38.341 INFO:teuthology.orchestra.run.smithi154.stdout:Total download size: 891 k 2024-04-02T04:47:38.341 INFO:teuthology.orchestra.run.smithi154.stdout:Installed size: 2.6 M 2024-04-02T04:47:38.341 INFO:teuthology.orchestra.run.smithi154.stdout:Downloading Packages: 2024-04-02T04:47:38.440 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:32 ago on Tue 02 Apr 2024 04:46:06 AM UTC. 2024-04-02T04:47:38.463 INFO:teuthology.orchestra.run.smithi154.stdout:(1/2): fuse-2.9.9-15.el9.x86_64.rpm 643 kB/s | 79 kB 00:00 2024-04-02T04:47:38.538 INFO:teuthology.orchestra.run.smithi152.stdout:Package librbd1-2:17.2.7-803.gc924f991.el9.x86_64 is already installed. 2024-04-02T04:47:38.590 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-02T04:47:38.591 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-02T04:47:38.591 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-02T04:47:38.658 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install python3-rados 2024-04-02T04:47:38.789 INFO:teuthology.orchestra.run.smithi154.stdout:(2/2): ceph-fuse-17.2.7-803.gc924f991.el9.x86_6 1.8 MB/s | 813 kB 00:00 2024-04-02T04:47:38.790 INFO:teuthology.orchestra.run.smithi154.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:47:38.790 INFO:teuthology.orchestra.run.smithi154.stdout:Total 1.9 MB/s | 891 kB 00:00 2024-04-02T04:47:38.790 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction check 2024-04-02T04:47:38.811 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction check succeeded. 2024-04-02T04:47:38.811 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction test 2024-04-02T04:47:38.884 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction test succeeded. 2024-04-02T04:47:38.884 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction 2024-04-02T04:47:39.065 INFO:teuthology.orchestra.run.smithi154.stdout: Preparing : 1/1 2024-04-02T04:47:39.192 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-04-02T04:47:39.227 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:33 ago on Tue 02 Apr 2024 04:46:06 AM UTC. 2024-04-02T04:47:39.256 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : ceph-fuse-2:17.2.7-803.gc924f991.el9.x86_64 2/2 2024-04-02T04:47:39.326 INFO:teuthology.orchestra.run.smithi152.stdout:Package python3-rados-2:17.2.7-803.gc924f991.el9.x86_64 is already installed. 2024-04-02T04:47:39.379 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-02T04:47:39.380 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-02T04:47:39.380 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-02T04:47:39.451 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install python3-rgw 2024-04-02T04:47:39.928 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: ceph-fuse-2:17.2.7-803.gc924f991.el9.x86_64 2/2 2024-04-02T04:47:39.928 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : ceph-fuse-2:17.2.7-803.gc924f991.el9.x86_64 1/2 2024-04-02T04:47:40.022 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:34 ago on Tue 02 Apr 2024 04:46:06 AM UTC. 2024-04-02T04:47:40.122 INFO:teuthology.orchestra.run.smithi152.stdout:Package python3-rgw-2:17.2.7-803.gc924f991.el9.x86_64 is already installed. 2024-04-02T04:47:40.176 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-02T04:47:40.177 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-02T04:47:40.178 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-02T04:47:40.242 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-04-02T04:47:40.242 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:40.242 INFO:teuthology.orchestra.run.smithi154.stdout:Installed: 2024-04-02T04:47:40.242 INFO:teuthology.orchestra.run.smithi154.stdout: ceph-fuse-2:17.2.7-803.gc924f991.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-04-02T04:47:40.242 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:40.242 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-04-02T04:47:40.250 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install python3-cephfs 2024-04-02T04:47:40.474 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum -y install librados-devel 2024-04-02T04:47:40.819 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:34 ago on Tue 02 Apr 2024 04:46:06 AM UTC. 2024-04-02T04:47:40.918 INFO:teuthology.orchestra.run.smithi152.stdout:Package python3-cephfs-2:17.2.7-803.gc924f991.el9.x86_64 is already installed. 2024-04-02T04:47:40.971 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-02T04:47:40.972 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-02T04:47:40.972 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-02T04:47:41.046 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install python3-rbd 2024-04-02T04:47:41.054 INFO:teuthology.orchestra.run.smithi154.stdout:Last metadata expiration check: 0:01:26 ago on Tue 02 Apr 2024 04:46:15 AM UTC. 2024-04-02T04:47:41.211 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-04-02T04:47:41.211 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:41.212 INFO:teuthology.orchestra.run.smithi154.stdout: Package Arch Version Repo Size 2024-04-02T04:47:41.212 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:41.212 INFO:teuthology.orchestra.run.smithi154.stdout:Installing: 2024-04-02T04:47:41.212 INFO:teuthology.orchestra.run.smithi154.stdout: librados-devel x86_64 2:17.2.7-803.gc924f991.el9 ceph 126 k 2024-04-02T04:47:41.212 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:41.212 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction Summary 2024-04-02T04:47:41.212 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:41.212 INFO:teuthology.orchestra.run.smithi154.stdout:Install 1 Package 2024-04-02T04:47:41.212 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:41.212 INFO:teuthology.orchestra.run.smithi154.stdout:Total download size: 126 k 2024-04-02T04:47:41.212 INFO:teuthology.orchestra.run.smithi154.stdout:Installed size: 448 k 2024-04-02T04:47:41.212 INFO:teuthology.orchestra.run.smithi154.stdout:Downloading Packages: 2024-04-02T04:47:41.443 INFO:teuthology.orchestra.run.smithi154.stdout:librados-devel-17.2.7-803.gc924f991.el9.x86_64. 548 kB/s | 126 kB 00:00 2024-04-02T04:47:41.444 INFO:teuthology.orchestra.run.smithi154.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:47:41.444 INFO:teuthology.orchestra.run.smithi154.stdout:Total 545 kB/s | 126 kB 00:00 2024-04-02T04:47:41.444 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction check 2024-04-02T04:47:41.449 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction check succeeded. 2024-04-02T04:47:41.450 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction test 2024-04-02T04:47:41.505 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction test succeeded. 2024-04-02T04:47:41.506 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction 2024-04-02T04:47:41.608 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:35 ago on Tue 02 Apr 2024 04:46:06 AM UTC. 2024-04-02T04:47:41.649 INFO:teuthology.orchestra.run.smithi154.stdout: Preparing : 1/1 2024-04-02T04:47:41.707 INFO:teuthology.orchestra.run.smithi152.stdout:Package python3-rbd-2:17.2.7-803.gc924f991.el9.x86_64 is already installed. 2024-04-02T04:47:41.750 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : librados-devel-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:41.761 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-02T04:47:41.762 INFO:teuthology.orchestra.run.smithi152.stdout:Nothing to do. 2024-04-02T04:47:41.762 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-02T04:47:41.836 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install rbd-fuse 2024-04-02T04:47:42.167 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: librados-devel-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:42.398 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:36 ago on Tue 02 Apr 2024 04:46:06 AM UTC. 2024-04-02T04:47:42.459 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : librados-devel-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:42.459 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:42.459 INFO:teuthology.orchestra.run.smithi154.stdout:Installed: 2024-04-02T04:47:42.459 INFO:teuthology.orchestra.run.smithi154.stdout: librados-devel-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:47:42.459 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:42.459 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-04-02T04:47:42.549 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-02T04:47:42.550 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:42.550 INFO:teuthology.orchestra.run.smithi152.stdout: Package Architecture Version Repository Size 2024-04-02T04:47:42.550 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:42.550 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-02T04:47:42.550 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-fuse x86_64 2:17.2.7-803.gc924f991.el9 ceph 82 k 2024-04-02T04:47:42.550 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:42.550 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-02T04:47:42.550 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:42.550 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-04-02T04:47:42.550 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:42.550 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 82 k 2024-04-02T04:47:42.551 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 222 k 2024-04-02T04:47:42.551 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-02T04:47:42.744 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum -y install libcephfs2 2024-04-02T04:47:42.863 INFO:teuthology.orchestra.run.smithi152.stdout:rbd-fuse-17.2.7-803.gc924f991.el9.x86_64.rpm 261 kB/s | 82 kB 00:00 2024-04-02T04:47:42.864 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:47:42.864 INFO:teuthology.orchestra.run.smithi152.stdout:Total 260 kB/s | 82 kB 00:00 2024-04-02T04:47:42.864 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-02T04:47:42.871 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-02T04:47:42.871 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-02T04:47:42.926 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-02T04:47:42.926 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-02T04:47:43.079 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-02T04:47:43.185 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : rbd-fuse-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:43.323 INFO:teuthology.orchestra.run.smithi154.stdout:Last metadata expiration check: 0:01:28 ago on Tue 02 Apr 2024 04:46:15 AM UTC. 2024-04-02T04:47:43.424 INFO:teuthology.orchestra.run.smithi154.stdout:Package libcephfs2-2:17.2.7-803.gc924f991.el9.x86_64 is already installed. 2024-04-02T04:47:43.481 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-04-02T04:47:43.482 INFO:teuthology.orchestra.run.smithi154.stdout:Nothing to do. 2024-04-02T04:47:43.482 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-04-02T04:47:43.552 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum -y install libcephfs-devel 2024-04-02T04:47:43.576 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: rbd-fuse-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:43.889 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : rbd-fuse-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:43.890 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:43.890 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-02T04:47:43.890 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-fuse-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:47:43.890 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:43.890 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-02T04:47:44.123 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install rbd-mirror 2024-04-02T04:47:44.125 INFO:teuthology.orchestra.run.smithi154.stdout:Last metadata expiration check: 0:01:29 ago on Tue 02 Apr 2024 04:46:15 AM UTC. 2024-04-02T04:47:44.276 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-04-02T04:47:44.277 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:44.277 INFO:teuthology.orchestra.run.smithi154.stdout: Package Arch Version Repo Size 2024-04-02T04:47:44.277 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:44.278 INFO:teuthology.orchestra.run.smithi154.stdout:Installing: 2024-04-02T04:47:44.278 INFO:teuthology.orchestra.run.smithi154.stdout: libcephfs-devel x86_64 2:17.2.7-803.gc924f991.el9 ceph 25 k 2024-04-02T04:47:44.278 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:44.278 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction Summary 2024-04-02T04:47:44.278 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:44.278 INFO:teuthology.orchestra.run.smithi154.stdout:Install 1 Package 2024-04-02T04:47:44.278 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:44.278 INFO:teuthology.orchestra.run.smithi154.stdout:Total download size: 25 k 2024-04-02T04:47:44.278 INFO:teuthology.orchestra.run.smithi154.stdout:Installed size: 110 k 2024-04-02T04:47:44.278 INFO:teuthology.orchestra.run.smithi154.stdout:Downloading Packages: 2024-04-02T04:47:44.446 INFO:teuthology.orchestra.run.smithi154.stdout:libcephfs-devel-17.2.7-803.gc924f991.el9.x86_64 149 kB/s | 25 kB 00:00 2024-04-02T04:47:44.447 INFO:teuthology.orchestra.run.smithi154.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:47:44.447 INFO:teuthology.orchestra.run.smithi154.stdout:Total 147 kB/s | 25 kB 00:00 2024-04-02T04:47:44.447 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction check 2024-04-02T04:47:44.450 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction check succeeded. 2024-04-02T04:47:44.450 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction test 2024-04-02T04:47:44.452 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction test succeeded. 2024-04-02T04:47:44.452 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction 2024-04-02T04:47:44.535 INFO:teuthology.orchestra.run.smithi154.stdout: Preparing : 1/1 2024-04-02T04:47:44.646 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : libcephfs-devel-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:44.700 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:38 ago on Tue 02 Apr 2024 04:46:06 AM UTC. 2024-04-02T04:47:44.860 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-02T04:47:44.861 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:44.861 INFO:teuthology.orchestra.run.smithi152.stdout: Package Arch Version Repo Size 2024-04-02T04:47:44.861 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:44.861 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-02T04:47:44.861 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-mirror x86_64 2:17.2.7-803.gc924f991.el9 ceph 2.9 M 2024-04-02T04:47:44.861 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:44.861 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-02T04:47:44.861 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:44.861 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-04-02T04:47:44.861 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:44.861 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 2.9 M 2024-04-02T04:47:44.861 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 12 M 2024-04-02T04:47:44.861 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-02T04:47:45.031 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:45.291 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : libcephfs-devel-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:45.292 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:45.292 INFO:teuthology.orchestra.run.smithi154.stdout:Installed: 2024-04-02T04:47:45.292 INFO:teuthology.orchestra.run.smithi154.stdout: libcephfs-devel-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:47:45.292 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:45.292 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-04-02T04:47:45.408 INFO:teuthology.orchestra.run.smithi152.stdout:rbd-mirror-17.2.7-803.gc924f991.el9.x86_64.rpm 5.4 MB/s | 2.9 MB 00:00 2024-04-02T04:47:45.408 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:47:45.408 INFO:teuthology.orchestra.run.smithi152.stdout:Total 5.4 MB/s | 2.9 MB 00:00 2024-04-02T04:47:45.409 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-02T04:47:45.418 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-02T04:47:45.418 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-02T04:47:45.471 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum -y install librados2 2024-04-02T04:47:45.489 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-02T04:47:45.489 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-02T04:47:45.896 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-02T04:47:45.958 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : rbd-mirror-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:45.982 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: rbd-mirror-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:45.982 INFO:teuthology.orchestra.run.smithi152.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-02T04:47:45.983 INFO:teuthology.orchestra.run.smithi152.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-04-02T04:47:45.983 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-02T04:47:45.983 INFO:teuthology.orchestra.run.smithi152.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-02T04:47:45.983 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:46.034 INFO:teuthology.orchestra.run.smithi154.stdout:Last metadata expiration check: 0:01:31 ago on Tue 02 Apr 2024 04:46:15 AM UTC. 2024-04-02T04:47:46.137 INFO:teuthology.orchestra.run.smithi154.stdout:Package librados2-2:17.2.7-803.gc924f991.el9.x86_64 is already installed. 2024-04-02T04:47:46.191 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-04-02T04:47:46.192 INFO:teuthology.orchestra.run.smithi154.stdout:Nothing to do. 2024-04-02T04:47:46.192 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-04-02T04:47:46.267 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum -y install librbd1 2024-04-02T04:47:46.828 INFO:teuthology.orchestra.run.smithi154.stdout:Last metadata expiration check: 0:01:31 ago on Tue 02 Apr 2024 04:46:15 AM UTC. 2024-04-02T04:47:46.927 INFO:teuthology.orchestra.run.smithi154.stdout:Package librbd1-2:17.2.7-803.gc924f991.el9.x86_64 is already installed. 2024-04-02T04:47:46.979 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-04-02T04:47:46.980 INFO:teuthology.orchestra.run.smithi154.stdout:Nothing to do. 2024-04-02T04:47:46.980 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-04-02T04:47:47.056 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum -y install python3-rados 2024-04-02T04:47:47.097 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : rbd-mirror-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:47.097 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:47.097 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-02T04:47:47.097 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-mirror-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:47:47.098 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:47.098 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-02T04:47:47.331 DEBUG:teuthology.orchestra.run.smithi152:> sudo yum -y install rbd-nbd 2024-04-02T04:47:47.623 INFO:teuthology.orchestra.run.smithi154.stdout:Last metadata expiration check: 0:01:32 ago on Tue 02 Apr 2024 04:46:15 AM UTC. 2024-04-02T04:47:47.723 INFO:teuthology.orchestra.run.smithi154.stdout:Package python3-rados-2:17.2.7-803.gc924f991.el9.x86_64 is already installed. 2024-04-02T04:47:47.776 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-04-02T04:47:47.777 INFO:teuthology.orchestra.run.smithi154.stdout:Nothing to do. 2024-04-02T04:47:47.777 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-04-02T04:47:47.849 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum -y install python3-rgw 2024-04-02T04:47:47.911 INFO:teuthology.orchestra.run.smithi152.stdout:Last metadata expiration check: 0:01:41 ago on Tue 02 Apr 2024 04:46:06 AM UTC. 2024-04-02T04:47:48.063 INFO:teuthology.orchestra.run.smithi152.stdout:Dependencies resolved. 2024-04-02T04:47:48.064 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:48.064 INFO:teuthology.orchestra.run.smithi152.stdout: Package Architecture Version Repository Size 2024-04-02T04:47:48.064 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:48.064 INFO:teuthology.orchestra.run.smithi152.stdout:Installing: 2024-04-02T04:47:48.064 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-nbd x86_64 2:17.2.7-803.gc924f991.el9 ceph 165 k 2024-04-02T04:47:48.064 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:48.064 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction Summary 2024-04-02T04:47:48.064 INFO:teuthology.orchestra.run.smithi152.stdout:================================================================================ 2024-04-02T04:47:48.064 INFO:teuthology.orchestra.run.smithi152.stdout:Install 1 Package 2024-04-02T04:47:48.064 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:48.064 INFO:teuthology.orchestra.run.smithi152.stdout:Total download size: 165 k 2024-04-02T04:47:48.064 INFO:teuthology.orchestra.run.smithi152.stdout:Installed size: 477 k 2024-04-02T04:47:48.064 INFO:teuthology.orchestra.run.smithi152.stdout:Downloading Packages: 2024-04-02T04:47:48.410 INFO:teuthology.orchestra.run.smithi152.stdout:rbd-nbd-17.2.7-803.gc924f991.el9.x86_64.rpm 478 kB/s | 165 kB 00:00 2024-04-02T04:47:48.411 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:47:48.411 INFO:teuthology.orchestra.run.smithi152.stdout:Total 476 kB/s | 165 kB 00:00 2024-04-02T04:47:48.411 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction check 2024-04-02T04:47:48.419 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction check succeeded. 2024-04-02T04:47:48.419 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction test 2024-04-02T04:47:48.419 INFO:teuthology.orchestra.run.smithi154.stdout:Last metadata expiration check: 0:01:33 ago on Tue 02 Apr 2024 04:46:15 AM UTC. 2024-04-02T04:47:48.476 INFO:teuthology.orchestra.run.smithi152.stdout:Transaction test succeeded. 2024-04-02T04:47:48.476 INFO:teuthology.orchestra.run.smithi152.stdout:Running transaction 2024-04-02T04:47:48.518 INFO:teuthology.orchestra.run.smithi154.stdout:Package python3-rgw-2:17.2.7-803.gc924f991.el9.x86_64 is already installed. 2024-04-02T04:47:48.572 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-04-02T04:47:48.573 INFO:teuthology.orchestra.run.smithi154.stdout:Nothing to do. 2024-04-02T04:47:48.573 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-04-02T04:47:48.652 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum -y install python3-cephfs 2024-04-02T04:47:48.658 INFO:teuthology.orchestra.run.smithi152.stdout: Preparing : 1/1 2024-04-02T04:47:48.767 INFO:teuthology.orchestra.run.smithi152.stdout: Installing : rbd-nbd-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:49.200 INFO:teuthology.orchestra.run.smithi152.stdout: Running scriptlet: rbd-nbd-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:49.218 INFO:teuthology.orchestra.run.smithi154.stdout:Last metadata expiration check: 0:01:34 ago on Tue 02 Apr 2024 04:46:15 AM UTC. 2024-04-02T04:47:49.317 INFO:teuthology.orchestra.run.smithi154.stdout:Package python3-cephfs-2:17.2.7-803.gc924f991.el9.x86_64 is already installed. 2024-04-02T04:47:49.369 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-04-02T04:47:49.370 INFO:teuthology.orchestra.run.smithi154.stdout:Nothing to do. 2024-04-02T04:47:49.370 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-04-02T04:47:49.447 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum -y install python3-rbd 2024-04-02T04:47:49.479 INFO:teuthology.orchestra.run.smithi152.stdout: Verifying : rbd-nbd-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:49.479 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:49.480 INFO:teuthology.orchestra.run.smithi152.stdout:Installed: 2024-04-02T04:47:49.480 INFO:teuthology.orchestra.run.smithi152.stdout: rbd-nbd-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:47:49.480 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:47:49.480 INFO:teuthology.orchestra.run.smithi152.stdout:Complete! 2024-04-02T04:47:49.723 DEBUG:teuthology.parallel:result is None 2024-04-02T04:47:50.016 INFO:teuthology.orchestra.run.smithi154.stdout:Last metadata expiration check: 0:01:35 ago on Tue 02 Apr 2024 04:46:15 AM UTC. 2024-04-02T04:47:50.115 INFO:teuthology.orchestra.run.smithi154.stdout:Package python3-rbd-2:17.2.7-803.gc924f991.el9.x86_64 is already installed. 2024-04-02T04:47:50.170 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-04-02T04:47:50.171 INFO:teuthology.orchestra.run.smithi154.stdout:Nothing to do. 2024-04-02T04:47:50.171 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-04-02T04:47:50.239 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum -y install rbd-fuse 2024-04-02T04:47:50.806 INFO:teuthology.orchestra.run.smithi154.stdout:Last metadata expiration check: 0:01:35 ago on Tue 02 Apr 2024 04:46:15 AM UTC. 2024-04-02T04:47:50.957 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-04-02T04:47:50.958 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:50.958 INFO:teuthology.orchestra.run.smithi154.stdout: Package Architecture Version Repository Size 2024-04-02T04:47:50.958 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:50.958 INFO:teuthology.orchestra.run.smithi154.stdout:Installing: 2024-04-02T04:47:50.958 INFO:teuthology.orchestra.run.smithi154.stdout: rbd-fuse x86_64 2:17.2.7-803.gc924f991.el9 ceph 82 k 2024-04-02T04:47:50.958 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:50.958 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction Summary 2024-04-02T04:47:50.958 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:50.958 INFO:teuthology.orchestra.run.smithi154.stdout:Install 1 Package 2024-04-02T04:47:50.958 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:50.959 INFO:teuthology.orchestra.run.smithi154.stdout:Total download size: 82 k 2024-04-02T04:47:50.959 INFO:teuthology.orchestra.run.smithi154.stdout:Installed size: 222 k 2024-04-02T04:47:50.959 INFO:teuthology.orchestra.run.smithi154.stdout:Downloading Packages: 2024-04-02T04:47:51.276 INFO:teuthology.orchestra.run.smithi154.stdout:rbd-fuse-17.2.7-803.gc924f991.el9.x86_64.rpm 257 kB/s | 82 kB 00:00 2024-04-02T04:47:51.277 INFO:teuthology.orchestra.run.smithi154.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:47:51.277 INFO:teuthology.orchestra.run.smithi154.stdout:Total 256 kB/s | 82 kB 00:00 2024-04-02T04:47:51.277 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction check 2024-04-02T04:47:51.284 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction check succeeded. 2024-04-02T04:47:51.284 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction test 2024-04-02T04:47:51.339 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction test succeeded. 2024-04-02T04:47:51.340 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction 2024-04-02T04:47:51.481 INFO:teuthology.orchestra.run.smithi154.stdout: Preparing : 1/1 2024-04-02T04:47:51.573 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : rbd-fuse-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:51.963 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: rbd-fuse-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:52.256 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : rbd-fuse-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:52.257 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:52.257 INFO:teuthology.orchestra.run.smithi154.stdout:Installed: 2024-04-02T04:47:52.257 INFO:teuthology.orchestra.run.smithi154.stdout: rbd-fuse-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:47:52.257 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:52.257 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-04-02T04:47:52.502 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum -y install rbd-mirror 2024-04-02T04:47:53.081 INFO:teuthology.orchestra.run.smithi154.stdout:Last metadata expiration check: 0:01:38 ago on Tue 02 Apr 2024 04:46:15 AM UTC. 2024-04-02T04:47:53.241 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-04-02T04:47:53.241 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:53.241 INFO:teuthology.orchestra.run.smithi154.stdout: Package Arch Version Repo Size 2024-04-02T04:47:53.241 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:53.242 INFO:teuthology.orchestra.run.smithi154.stdout:Installing: 2024-04-02T04:47:53.242 INFO:teuthology.orchestra.run.smithi154.stdout: rbd-mirror x86_64 2:17.2.7-803.gc924f991.el9 ceph 2.9 M 2024-04-02T04:47:53.242 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:53.242 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction Summary 2024-04-02T04:47:53.242 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:53.242 INFO:teuthology.orchestra.run.smithi154.stdout:Install 1 Package 2024-04-02T04:47:53.242 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:53.243 INFO:teuthology.orchestra.run.smithi154.stdout:Total download size: 2.9 M 2024-04-02T04:47:53.243 INFO:teuthology.orchestra.run.smithi154.stdout:Installed size: 12 M 2024-04-02T04:47:53.243 INFO:teuthology.orchestra.run.smithi154.stdout:Downloading Packages: 2024-04-02T04:47:53.621 INFO:teuthology.orchestra.run.smithi154.stdout:rbd-mirror-17.2.7-803.gc924f991.el9.x86_64.rpm 7.8 MB/s | 2.9 MB 00:00 2024-04-02T04:47:53.621 INFO:teuthology.orchestra.run.smithi154.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:47:53.621 INFO:teuthology.orchestra.run.smithi154.stdout:Total 7.7 MB/s | 2.9 MB 00:00 2024-04-02T04:47:53.622 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction check 2024-04-02T04:47:53.631 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction check succeeded. 2024-04-02T04:47:53.631 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction test 2024-04-02T04:47:53.703 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction test succeeded. 2024-04-02T04:47:53.703 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction 2024-04-02T04:47:54.110 INFO:teuthology.orchestra.run.smithi154.stdout: Preparing : 1/1 2024-04-02T04:47:54.179 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : rbd-mirror-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:54.204 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: rbd-mirror-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:54.204 INFO:teuthology.orchestra.run.smithi154.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-04-02T04:47:54.204 INFO:teuthology.orchestra.run.smithi154.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-04-02T04:47:54.204 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-02T04:47:54.204 INFO:teuthology.orchestra.run.smithi154.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-04-02T04:47:54.204 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:55.326 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : rbd-mirror-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:55.327 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:55.327 INFO:teuthology.orchestra.run.smithi154.stdout:Installed: 2024-04-02T04:47:55.327 INFO:teuthology.orchestra.run.smithi154.stdout: rbd-mirror-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:47:55.327 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:55.327 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-04-02T04:47:55.548 DEBUG:teuthology.orchestra.run.smithi154:> sudo yum -y install rbd-nbd 2024-04-02T04:47:56.128 INFO:teuthology.orchestra.run.smithi154.stdout:Last metadata expiration check: 0:01:41 ago on Tue 02 Apr 2024 04:46:15 AM UTC. 2024-04-02T04:47:56.279 INFO:teuthology.orchestra.run.smithi154.stdout:Dependencies resolved. 2024-04-02T04:47:56.281 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:56.281 INFO:teuthology.orchestra.run.smithi154.stdout: Package Architecture Version Repository Size 2024-04-02T04:47:56.281 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:56.281 INFO:teuthology.orchestra.run.smithi154.stdout:Installing: 2024-04-02T04:47:56.281 INFO:teuthology.orchestra.run.smithi154.stdout: rbd-nbd x86_64 2:17.2.7-803.gc924f991.el9 ceph 165 k 2024-04-02T04:47:56.281 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:56.281 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction Summary 2024-04-02T04:47:56.282 INFO:teuthology.orchestra.run.smithi154.stdout:================================================================================ 2024-04-02T04:47:56.282 INFO:teuthology.orchestra.run.smithi154.stdout:Install 1 Package 2024-04-02T04:47:56.282 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:56.282 INFO:teuthology.orchestra.run.smithi154.stdout:Total download size: 165 k 2024-04-02T04:47:56.282 INFO:teuthology.orchestra.run.smithi154.stdout:Installed size: 477 k 2024-04-02T04:47:56.282 INFO:teuthology.orchestra.run.smithi154.stdout:Downloading Packages: 2024-04-02T04:47:56.515 INFO:teuthology.orchestra.run.smithi154.stdout:rbd-nbd-17.2.7-803.gc924f991.el9.x86_64.rpm 705 kB/s | 165 kB 00:00 2024-04-02T04:47:56.516 INFO:teuthology.orchestra.run.smithi154.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:47:56.516 INFO:teuthology.orchestra.run.smithi154.stdout:Total 700 kB/s | 165 kB 00:00 2024-04-02T04:47:56.516 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction check 2024-04-02T04:47:56.524 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction check succeeded. 2024-04-02T04:47:56.524 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction test 2024-04-02T04:47:56.579 INFO:teuthology.orchestra.run.smithi154.stdout:Transaction test succeeded. 2024-04-02T04:47:56.579 INFO:teuthology.orchestra.run.smithi154.stdout:Running transaction 2024-04-02T04:47:56.725 INFO:teuthology.orchestra.run.smithi154.stdout: Preparing : 1/1 2024-04-02T04:47:56.837 INFO:teuthology.orchestra.run.smithi154.stdout: Installing : rbd-nbd-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:57.279 INFO:teuthology.orchestra.run.smithi154.stdout: Running scriptlet: rbd-nbd-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:57.572 INFO:teuthology.orchestra.run.smithi154.stdout: Verifying : rbd-nbd-2:17.2.7-803.gc924f991.el9.x86_64 1/1 2024-04-02T04:47:57.572 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:57.572 INFO:teuthology.orchestra.run.smithi154.stdout:Installed: 2024-04-02T04:47:57.572 INFO:teuthology.orchestra.run.smithi154.stdout: rbd-nbd-2:17.2.7-803.gc924f991.el9.x86_64 2024-04-02T04:47:57.573 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:47:57.573 INFO:teuthology.orchestra.run.smithi154.stdout:Complete! 2024-04-02T04:47:57.801 DEBUG:teuthology.parallel:result is None 2024-04-02T04:47:57.801 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-02T04:47:57.801 INFO:teuthology.packaging:ref: None 2024-04-02T04:47:57.801 INFO:teuthology.packaging:tag: None 2024-04-02T04:47:57.801 INFO:teuthology.packaging:branch: quincy 2024-04-02T04:47:57.802 INFO:teuthology.packaging:sha1: 6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T04:47:57.802 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-02T04:47:58.030 DEBUG:teuthology.orchestra.run.smithi152:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-02T04:47:58.056 INFO:teuthology.orchestra.run.smithi152.stdout:17.2.7-803.gc924f991.el9 2024-04-02T04:47:58.058 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-803.gc924f991.el9 2024-04-02T04:47:58.058 INFO:teuthology.task.install:The correct ceph version 17.2.7-803.gc924f991 is installed. 2024-04-02T04:47:58.060 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-02T04:47:58.060 INFO:teuthology.packaging:ref: None 2024-04-02T04:47:58.060 INFO:teuthology.packaging:tag: None 2024-04-02T04:47:58.060 INFO:teuthology.packaging:branch: quincy 2024-04-02T04:47:58.061 INFO:teuthology.packaging:sha1: 6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T04:47:58.061 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-04-02T04:47:58.372 DEBUG:teuthology.orchestra.run.smithi154:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-04-02T04:47:58.395 INFO:teuthology.orchestra.run.smithi154.stdout:17.2.7-803.gc924f991.el9 2024-04-02T04:47:58.395 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-803.gc924f991.el9 2024-04-02T04:47:58.396 INFO:teuthology.task.install:The correct ceph version 17.2.7-803.gc924f991 is installed. 2024-04-02T04:47:58.397 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-04-02T04:47:58.398 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-02T04:47:58.398 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-02T04:47:58.431 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-04-02T04:47:58.431 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-02T04:47:58.493 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-04-02T04:47:58.493 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-02T04:47:58.493 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/usr/bin/daemon-helper 2024-04-02T04:47:58.523 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-02T04:47:58.593 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-04-02T04:47:58.593 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd of=/usr/bin/daemon-helper 2024-04-02T04:47:58.621 DEBUG:teuthology.orchestra.run.smithi154:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-02T04:47:58.691 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-04-02T04:47:58.691 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-02T04:47:58.691 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-02T04:47:58.720 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-02T04:47:58.790 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-04-02T04:47:58.790 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-02T04:47:58.818 DEBUG:teuthology.orchestra.run.smithi154:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-02T04:47:58.885 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-04-02T04:47:58.885 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-02T04:47:58.885 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/usr/bin/stdin-killer 2024-04-02T04:47:58.915 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-02T04:47:58.983 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-04-02T04:47:58.983 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd of=/usr/bin/stdin-killer 2024-04-02T04:47:59.010 DEBUG:teuthology.orchestra.run.smithi154:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-02T04:47:59.077 INFO:teuthology.run_tasks:Running task print... 2024-04-02T04:47:59.086 INFO:teuthology.task.print:**** done install task... 2024-04-02T04:47:59.086 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-02T04:47:59.187 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore', 'osd op complaint time': 180}, 'client': {'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'global': {'mon pg warn min per osd': 0}, 'mds': {'debug mds': 20, 'debug mds balancer': 20, 'debug ms': 1, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mgr': {'debug client': 20, 'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon down mkfs grace': 300, 'mon op complaint time': 120}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'roleless': True, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'FS_DEGRADED', 'FS_INLINE_DATA_DEPRECATED', 'FS_WITH_FAILED_MDS', 'MDS_ALL_DOWN', 'MDS_DAMAGE', 'MDS_DEGRADED', 'MDS_FAILED', 'MDS_INSUFFICIENT_STANDBY', 'MDS_UP_LESS_THAN_MAX', 'POOL_APP_NOT_ENABLED', 'overall HEALTH_', 'Replacing daemon', 'OSD_DOWN', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding'], 'sha1': '6cec793d9073f0594e2364b9b687ecaf22424cca'} 2024-04-02T04:47:59.187 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-04-02T04:47:59.188 INFO:tasks.cephadm:Cluster fsid is 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 2024-04-02T04:47:59.188 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-02T04:47:59.188 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-04-02T04:47:59.188 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi152': '172.21.15.152', 'mon.smithi154': '172.21.15.154'} 2024-04-02T04:47:59.188 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-02T04:47:59.188 DEBUG:teuthology.orchestra.run.smithi152:> sudo hostname $(hostname -s) 2024-04-02T04:47:59.218 DEBUG:teuthology.orchestra.run.smithi154:> sudo hostname $(hostname -s) 2024-04-02T04:47:59.245 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-04-02T04:47:59.246 DEBUG:teuthology.orchestra.run.smithi152:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-02T04:47:59.503 INFO:teuthology.orchestra.run.smithi152.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Apr 2 04:47 /home/ubuntu/cephtest/cephadm 2024-04-02T04:47:59.503 DEBUG:teuthology.orchestra.run.smithi154:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-02T04:47:59.766 INFO:teuthology.orchestra.run.smithi154.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Apr 2 04:47 /home/ubuntu/cephtest/cephadm 2024-04-02T04:47:59.767 DEBUG:teuthology.orchestra.run.smithi152:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-02T04:47:59.784 DEBUG:teuthology.orchestra.run.smithi154:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-02T04:47:59.809 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-04-02T04:47:59.810 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-04-02T04:47:59.827 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-04-02T04:48:00.653 INFO:teuthology.orchestra.run.smithi152.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-02T04:48:00.735 INFO:teuthology.orchestra.run.smithi154.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-02T04:48:44.423 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-04-02T04:48:44.423 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph_version": "ceph version 17.2.7-803-gc924f991 (c924f9914f2809997b363f77c5cbb14929881220) quincy (stable)", 2024-04-02T04:48:44.423 INFO:teuthology.orchestra.run.smithi152.stdout: "image_id": "4948510fac0f2ac7df4b4e361573714a48c110d48535a4b8d5aeb81e8023bedd", 2024-04-02T04:48:44.423 INFO:teuthology.orchestra.run.smithi152.stdout: "repo_digests": [ 2024-04-02T04:48:44.423 INFO:teuthology.orchestra.run.smithi152.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:afcab66d4788c00ea37075a6986aaec87a4f099e0d803aec3d45584d18bd8a6e" 2024-04-02T04:48:44.423 INFO:teuthology.orchestra.run.smithi152.stdout: ] 2024-04-02T04:48:44.423 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-04-02T04:48:51.215 INFO:teuthology.orchestra.run.smithi154.stdout:{ 2024-04-02T04:48:51.216 INFO:teuthology.orchestra.run.smithi154.stdout: "ceph_version": "ceph version 17.2.7-803-gc924f991 (c924f9914f2809997b363f77c5cbb14929881220) quincy (stable)", 2024-04-02T04:48:51.216 INFO:teuthology.orchestra.run.smithi154.stdout: "image_id": "4948510fac0f2ac7df4b4e361573714a48c110d48535a4b8d5aeb81e8023bedd", 2024-04-02T04:48:51.216 INFO:teuthology.orchestra.run.smithi154.stdout: "repo_digests": [ 2024-04-02T04:48:51.216 INFO:teuthology.orchestra.run.smithi154.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:afcab66d4788c00ea37075a6986aaec87a4f099e0d803aec3d45584d18bd8a6e" 2024-04-02T04:48:51.216 INFO:teuthology.orchestra.run.smithi154.stdout: ] 2024-04-02T04:48:51.216 INFO:teuthology.orchestra.run.smithi154.stdout:} 2024-04-02T04:48:51.234 DEBUG:teuthology.orchestra.run.smithi152:> sudo mkdir -p /etc/ceph 2024-04-02T04:48:51.268 DEBUG:teuthology.orchestra.run.smithi154:> sudo mkdir -p /etc/ceph 2024-04-02T04:48:51.297 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod 777 /etc/ceph 2024-04-02T04:48:51.339 DEBUG:teuthology.orchestra.run.smithi154:> sudo chmod 777 /etc/ceph 2024-04-02T04:48:51.365 INFO:tasks.cephadm:Writing seed config... 2024-04-02T04:48:51.367 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-04-02T04:48:51.367 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-04-02T04:48:51.367 INFO:tasks.cephadm: override: [osd] bdev async discard = True 2024-04-02T04:48:51.367 INFO:tasks.cephadm: override: [osd] bdev enable discard = True 2024-04-02T04:48:51.367 INFO:tasks.cephadm: override: [osd] bluestore allocator = bitmap 2024-04-02T04:48:51.367 INFO:tasks.cephadm: override: [osd] bluestore block size = 96636764160 2024-04-02T04:48:51.367 INFO:tasks.cephadm: override: [osd] bluestore fsck on mount = True 2024-04-02T04:48:51.367 INFO:tasks.cephadm: override: [osd] debug bluefs = 1/20 2024-04-02T04:48:51.367 INFO:tasks.cephadm: override: [osd] debug bluestore = 1/20 2024-04-02T04:48:51.367 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-02T04:48:51.367 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-02T04:48:51.367 INFO:tasks.cephadm: override: [osd] debug rocksdb = 4/10 2024-04-02T04:48:51.367 INFO:tasks.cephadm: override: [osd] mon osd backfillfull_ratio = 0.85 2024-04-02T04:48:51.368 INFO:tasks.cephadm: override: [osd] mon osd full ratio = 0.9 2024-04-02T04:48:51.368 INFO:tasks.cephadm: override: [osd] mon osd nearfull ratio = 0.8 2024-04-02T04:48:51.368 INFO:tasks.cephadm: override: [osd] osd failsafe full ratio = 0.95 2024-04-02T04:48:51.368 INFO:tasks.cephadm: override: [osd] osd objectstore = bluestore 2024-04-02T04:48:51.368 INFO:tasks.cephadm: override: [osd] osd op complaint time = 180 2024-04-02T04:48:51.368 INFO:tasks.cephadm: override: [client] client mount timeout = 600 2024-04-02T04:48:51.368 INFO:tasks.cephadm: override: [client] debug client = 20 2024-04-02T04:48:51.368 INFO:tasks.cephadm: override: [client] debug ms = 1 2024-04-02T04:48:51.368 INFO:tasks.cephadm: override: [client] rados mon op timeout = 900 2024-04-02T04:48:51.368 INFO:tasks.cephadm: override: [client] rados osd op timeout = 900 2024-04-02T04:48:51.368 INFO:tasks.cephadm: override: [global] mon pg warn min per osd = 0 2024-04-02T04:48:51.368 INFO:tasks.cephadm: override: [mds] debug mds = 20 2024-04-02T04:48:51.368 INFO:tasks.cephadm: override: [mds] debug mds balancer = 20 2024-04-02T04:48:51.368 INFO:tasks.cephadm: override: [mds] debug ms = 1 2024-04-02T04:48:51.368 INFO:tasks.cephadm: override: [mds] mds debug frag = True 2024-04-02T04:48:51.369 INFO:tasks.cephadm: override: [mds] mds debug scatterstat = True 2024-04-02T04:48:51.369 INFO:tasks.cephadm: override: [mds] mds op complaint time = 180 2024-04-02T04:48:51.369 INFO:tasks.cephadm: override: [mds] mds verify scatter = True 2024-04-02T04:48:51.369 INFO:tasks.cephadm: override: [mds] osd op complaint time = 180 2024-04-02T04:48:51.369 INFO:tasks.cephadm: override: [mds] rados mon op timeout = 900 2024-04-02T04:48:51.369 INFO:tasks.cephadm: override: [mds] rados osd op timeout = 900 2024-04-02T04:48:51.369 INFO:tasks.cephadm: override: [mgr] debug client = 20 2024-04-02T04:48:51.369 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-02T04:48:51.369 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-02T04:48:51.369 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-02T04:48:51.369 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-02T04:48:51.369 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-02T04:48:51.369 INFO:tasks.cephadm: override: [mon] mon down mkfs grace = 300 2024-04-02T04:48:51.369 INFO:tasks.cephadm: override: [mon] mon op complaint time = 120 2024-04-02T04:48:51.370 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-02T04:48:51.370 DEBUG:teuthology.orchestra.run.smithi152:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-02T04:48:51.395 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 mon pg warn min per osd = 0 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * bdev async discard = True bdev enable discard = True bluestore allocator = bitmap bluestore block size = 96636764160 bluestore fsck on mount = True debug bluefs = 1/20 debug bluestore = 1/20 debug ms = 1 debug osd = 20 debug rocksdb = 4/10 mon osd backfillfull_ratio = 0.85 mon osd full ratio = 0.9 mon osd nearfull ratio = 0.8 osd failsafe full ratio = 0.95 osd objectstore = bluestore osd op complaint time = 180 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug client = 20 debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 mon down mkfs grace = 300 mon op complaint time = 120 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true [client] client mount timeout = 600 debug client = 20 debug ms = 1 rados mon op timeout = 900 rados osd op timeout = 900 [mds] debug mds = 20 debug mds balancer = 20 debug ms = 1 mds debug frag = True mds debug scatterstat = True mds op complaint time = 180 mds verify scatter = True osd op complaint time = 180 rados mon op timeout = 900 rados osd op timeout = 900 2024-04-02T04:48:51.396 DEBUG:teuthology.orchestra.run.smithi152:mon.smithi152> sudo journalctl -f -n 0 -u ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4@mon.smithi152.service 2024-04-02T04:48:51.438 INFO:tasks.cephadm:Bootstrapping... 2024-04-02T04:48:51.439 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.152 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-02T04:48:51.646 INFO:teuthology.orchestra.run.smithi152.stdout:-------------------------------------------------------------------------------- 2024-04-02T04:48:51.646 INFO:teuthology.orchestra.run.smithi152.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', '2e0055d0-f0ac-11ee-b647-cb9ed24678a4', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.152', '--skip-admin-label'] 2024-04-02T04:48:51.671 INFO:teuthology.orchestra.run.smithi152.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-02T04:48:51.671 INFO:teuthology.orchestra.run.smithi152.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-04-02T04:48:51.672 INFO:teuthology.orchestra.run.smithi152.stdout:Verifying podman|docker is present... 2024-04-02T04:48:51.695 INFO:teuthology.orchestra.run.smithi152.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-02T04:48:51.695 INFO:teuthology.orchestra.run.smithi152.stdout:Verifying lvm2 is present... 2024-04-02T04:48:51.696 INFO:teuthology.orchestra.run.smithi152.stdout:Verifying time synchronization is in place... 2024-04-02T04:48:51.703 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-02T04:48:51.704 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-02T04:48:51.710 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-02T04:48:51.710 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout inactive 2024-04-02T04:48:51.716 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout enabled 2024-04-02T04:48:51.723 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout active 2024-04-02T04:48:51.723 INFO:teuthology.orchestra.run.smithi152.stdout:Unit chronyd.service is enabled and running 2024-04-02T04:48:51.723 INFO:teuthology.orchestra.run.smithi152.stdout:Repeating the final host check... 2024-04-02T04:48:51.746 INFO:teuthology.orchestra.run.smithi152.stdout:/bin/podman: stdout 4.9.4-dev 2024-04-02T04:48:51.746 INFO:teuthology.orchestra.run.smithi152.stdout:podman (/bin/podman) version 4.9.4 is present 2024-04-02T04:48:51.746 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl is present 2024-04-02T04:48:51.746 INFO:teuthology.orchestra.run.smithi152.stdout:lvcreate is present 2024-04-02T04:48:51.753 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-02T04:48:51.753 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-02T04:48:51.759 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-02T04:48:51.759 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout inactive 2024-04-02T04:48:51.765 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout enabled 2024-04-02T04:48:51.772 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout active 2024-04-02T04:48:51.772 INFO:teuthology.orchestra.run.smithi152.stdout:Unit chronyd.service is enabled and running 2024-04-02T04:48:51.772 INFO:teuthology.orchestra.run.smithi152.stdout:Host looks OK 2024-04-02T04:48:51.772 INFO:teuthology.orchestra.run.smithi152.stdout:Cluster fsid: 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 2024-04-02T04:48:51.772 INFO:teuthology.orchestra.run.smithi152.stdout:Acquiring lock 140414870287456 on /run/cephadm/2e0055d0-f0ac-11ee-b647-cb9ed24678a4.lock 2024-04-02T04:48:51.772 INFO:teuthology.orchestra.run.smithi152.stdout:Lock 140414870287456 acquired on /run/cephadm/2e0055d0-f0ac-11ee-b647-cb9ed24678a4.lock 2024-04-02T04:48:51.773 INFO:teuthology.orchestra.run.smithi152.stdout:Verifying IP 172.21.15.152 port 3300 ... 2024-04-02T04:48:51.773 INFO:teuthology.orchestra.run.smithi152.stdout:Verifying IP 172.21.15.152 port 6789 ... 2024-04-02T04:48:51.773 INFO:teuthology.orchestra.run.smithi152.stdout:Base mon IP(s) is [172.21.15.152:3300, 172.21.15.152:6789], mon addrv is [v2:172.21.15.152:3300,v1:172.21.15.152:6789] 2024-04-02T04:48:51.776 INFO:teuthology.orchestra.run.smithi152.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.152 metric 100 2024-04-02T04:48:51.776 INFO:teuthology.orchestra.run.smithi152.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.152 metric 100 2024-04-02T04:48:51.779 INFO:teuthology.orchestra.run.smithi152.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-04-02T04:48:51.779 INFO:teuthology.orchestra.run.smithi152.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-04-02T04:48:51.779 INFO:teuthology.orchestra.run.smithi152.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1735sec hoplimit 64 pref medium 2024-04-02T04:48:51.782 INFO:teuthology.orchestra.run.smithi152.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-02T04:48:51.782 INFO:teuthology.orchestra.run.smithi152.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-04-02T04:48:51.782 INFO:teuthology.orchestra.run.smithi152.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-02T04:48:51.782 INFO:teuthology.orchestra.run.smithi152.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-04-02T04:48:51.782 INFO:teuthology.orchestra.run.smithi152.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:fe88:6de3/64 scope link 2024-04-02T04:48:51.782 INFO:teuthology.orchestra.run.smithi152.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-02T04:48:51.783 INFO:teuthology.orchestra.run.smithi152.stdout:Mon IP `172.21.15.152` is in CIDR network `172.21.0.0/20` 2024-04-02T04:48:51.783 INFO:teuthology.orchestra.run.smithi152.stdout:Mon IP `172.21.15.152` is in CIDR network `172.21.0.0/20` 2024-04-02T04:48:51.783 INFO:teuthology.orchestra.run.smithi152.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-04-02T04:48:51.783 INFO:teuthology.orchestra.run.smithi152.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-02T04:48:51.784 INFO:teuthology.orchestra.run.smithi152.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-02T04:48:52.264 INFO:teuthology.orchestra.run.smithi152.stdout:/bin/podman: stdout 4948510fac0f2ac7df4b4e361573714a48c110d48535a4b8d5aeb81e8023bedd 2024-04-02T04:48:52.264 INFO:teuthology.orchestra.run.smithi152.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2024-04-02T04:48:52.264 INFO:teuthology.orchestra.run.smithi152.stdout:/bin/podman: stderr Getting image source signatures 2024-04-02T04:48:52.264 INFO:teuthology.orchestra.run.smithi152.stdout:/bin/podman: stderr Copying blob sha256:da4fb30068f211e3374605e92d29aa994b50548e89df01b7dc881a42ea810f22 2024-04-02T04:48:52.264 INFO:teuthology.orchestra.run.smithi152.stdout:/bin/podman: stderr Copying blob sha256:11da15500eca1f826c328eb3265caa30b0a3730510e2a32023084d6fe9cee20c 2024-04-02T04:48:52.264 INFO:teuthology.orchestra.run.smithi152.stdout:/bin/podman: stderr Copying config sha256:4948510fac0f2ac7df4b4e361573714a48c110d48535a4b8d5aeb81e8023bedd 2024-04-02T04:48:52.264 INFO:teuthology.orchestra.run.smithi152.stdout:/bin/podman: stderr Writing manifest to image destination 2024-04-02T04:48:53.084 INFO:teuthology.orchestra.run.smithi152.stdout:ceph: stdout ceph version 17.2.7-803-gc924f991 (c924f9914f2809997b363f77c5cbb14929881220) quincy (stable) 2024-04-02T04:48:53.084 INFO:teuthology.orchestra.run.smithi152.stdout:Ceph version: ceph version 17.2.7-803-gc924f991 (c924f9914f2809997b363f77c5cbb14929881220) quincy (stable) 2024-04-02T04:48:53.085 INFO:teuthology.orchestra.run.smithi152.stdout:Extracting ceph user uid/gid from container image... 2024-04-02T04:48:54.057 INFO:teuthology.orchestra.run.smithi152.stdout:stat: stdout 167 167 2024-04-02T04:48:54.057 INFO:teuthology.orchestra.run.smithi152.stdout:Creating initial keys... 2024-04-02T04:48:54.914 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph-authtool: stdout AQA2jgtmy4JxExAAn5Bi8wBcDdr9XRQPsr7jJQ== 2024-04-02T04:48:55.754 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph-authtool: stdout AQA3jgtmFn+ADBAALJpURDemnyV5WuX88Nc5Hg== 2024-04-02T04:48:56.553 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph-authtool: stdout AQA4jgtmWqrzAxAAHufueIN0EP3iwZSLRHAqfA== 2024-04-02T04:48:56.554 INFO:teuthology.orchestra.run.smithi152.stdout:Creating initial monmap... 2024-04-02T04:48:57.493 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-02T04:48:57.493 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-04-02T04:48:57.493 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 2024-04-02T04:48:57.493 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-02T04:48:57.493 INFO:teuthology.orchestra.run.smithi152.stdout:monmaptool for smithi152 [v2:172.21.15.152:3300,v1:172.21.15.152:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-02T04:48:57.493 INFO:teuthology.orchestra.run.smithi152.stdout:setting min_mon_release = octopus 2024-04-02T04:48:57.493 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/monmaptool: set fsid to 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 2024-04-02T04:48:57.493 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-02T04:48:57.493 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:48:57.493 INFO:teuthology.orchestra.run.smithi152.stdout:Creating mon... 2024-04-02T04:48:58.390 INFO:teuthology.orchestra.run.smithi152.stdout:create mon.smithi152 on 2024-04-02T04:48:58.588 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-04-02T04:48:58.750 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-02T04:48:58.906 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4.target → /etc/systemd/system/ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4.target. 2024-04-02T04:48:58.906 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4.target → /etc/systemd/system/ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4.target. 2024-04-02T04:48:59.092 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4@mon.smithi152 2024-04-02T04:48:59.092 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stderr Failed to reset failed state of unit ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4@mon.smithi152.service: Unit ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4@mon.smithi152.service not loaded. 2024-04-02T04:48:59.279 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4.target.wants/ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4@mon.smithi152.service → /etc/systemd/system/ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4@.service. 2024-04-02T04:48:59.601 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:48:59 smithi152 podman[28681]: 2024-04-02 04:48:59.430646469 +0000 UTC m=+0.018165739 image pull 4948510fac0f2ac7df4b4e361573714a48c110d48535a4b8d5aeb81e8023bedd quay.ceph.io/ceph-ci/ceph:quincy 2024-04-02T04:48:59.814 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-02T04:48:59.814 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout disabled 2024-04-02T04:48:59.821 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-02T04:48:59.821 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout inactive 2024-04-02T04:48:59.821 INFO:teuthology.orchestra.run.smithi152.stdout:firewalld.service is not enabled 2024-04-02T04:48:59.821 INFO:teuthology.orchestra.run.smithi152.stdout:Not possible to enable service . firewalld.service is not available 2024-04-02T04:48:59.821 INFO:teuthology.orchestra.run.smithi152.stdout:Waiting for mon to start... 2024-04-02T04:48:59.821 INFO:teuthology.orchestra.run.smithi152.stdout:Waiting for mon... 2024-04-02T04:48:59.884 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:48:59 smithi152 podman[28681]: 2024-04-02 04:48:59.706165379 +0000 UTC m=+0.293684646 container init 334f706d66e0a0611e54df430be3c7b875dcb026e3c798c242f63bebb90b7288 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-mon-smithi152, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, ceph=True, RELEASE=quincy-c924f99, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-02T04:48:59.884 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:48:59 smithi152 podman[28681]: 2024-04-02 04:48:59.709117587 +0000 UTC m=+0.296636854 container start 334f706d66e0a0611e54df430be3c7b875dcb026e3c798c242f63bebb90b7288 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-mon-smithi152, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, RELEASE=quincy-c924f99, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-02T04:48:59.884 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:48:59 smithi152 ceph-mon[28699]: mkfs 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 2024-04-02T04:48:59.885 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:48:59 smithi152 ceph-mon[28699]: mon.smithi152 is new leader, mons smithi152 in quorum (ranks 0) 2024-04-02T04:48:59.885 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:48:59 smithi152 bash[28681]: 334f706d66e0a0611e54df430be3c7b875dcb026e3c798c242f63bebb90b7288 2024-04-02T04:48:59.885 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:48:59 smithi152 systemd[1]: Started Ceph mon.smithi152 for 2e0055d0-f0ac-11ee-b647-cb9ed24678a4. 2024-04-02T04:49:00.866 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout cluster: 2024-04-02T04:49:00.866 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout id: 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 2024-04-02T04:49:00.866 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-04-02T04:49:00.866 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 2024-04-02T04:49:00.867 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout services: 2024-04-02T04:49:00.867 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum smithi152 (age 0.562215s) 2024-04-02T04:49:00.867 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-04-02T04:49:00.867 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-04-02T04:49:00.867 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 2024-04-02T04:49:00.867 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout data: 2024-04-02T04:49:00.867 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-04-02T04:49:00.867 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-04-02T04:49:00.867 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-04-02T04:49:00.867 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout pgs: 2024-04-02T04:49:00.867 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 2024-04-02T04:49:00.867 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.229+0000 7f024ebfe700 1 Processor -- start 2024-04-02T04:49:00.867 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.229+0000 7f024ebfe700 1 -- start start 2024-04-02T04:49:00.867 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.229+0000 7f024ebfe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0248076c70 0x7f0248105440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:00.867 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.229+0000 7f024ebfe700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0248105980 con 0x7f0248076c70 2024-04-02T04:49:00.867 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.230+0000 7f024c99a700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0248076c70 0x7f0248105440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:00.868 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.230+0000 7f024c99a700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0248076c70 0x7f0248105440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:55426/0 (socket says 172.21.15.152:55426) 2024-04-02T04:49:00.868 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.230+0000 7f024c99a700 1 -- 172.21.15.152:0/3479618295 learned_addr learned my addr 172.21.15.152:0/3479618295 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:00.868 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.230+0000 7f024c99a700 1 -- 172.21.15.152:0/3479618295 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0248105ac0 con 0x7f0248076c70 2024-04-02T04:49:00.868 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.230+0000 7f024c99a700 1 --2- 172.21.15.152:0/3479618295 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0248076c70 0x7f0248105440 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f023c0098e0 tx=0x7f023c009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=616c4c45945748c4 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:00.868 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.230+0000 7f02477fe700 1 -- 172.21.15.152:0/3479618295 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f023c00b860 con 0x7f0248076c70 2024-04-02T04:49:00.868 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.231+0000 7f02477fe700 1 -- 172.21.15.152:0/3479618295 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f023c00b9c0 con 0x7f0248076c70 2024-04-02T04:49:00.868 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.231+0000 7f02477fe700 1 -- 172.21.15.152:0/3479618295 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f023c00bc90 con 0x7f0248076c70 2024-04-02T04:49:00.868 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.231+0000 7f024ebfe700 1 -- 172.21.15.152:0/3479618295 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0248076c70 msgr2=0x7f0248105440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:00.868 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.231+0000 7f024ebfe700 1 --2- 172.21.15.152:0/3479618295 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0248076c70 0x7f0248105440 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f023c0098e0 tx=0x7f023c009250 comp rx=0 tx=0).stop 2024-04-02T04:49:00.868 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.231+0000 7f024ebfe700 1 -- 172.21.15.152:0/3479618295 shutdown_connections 2024-04-02T04:49:00.868 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.231+0000 7f024ebfe700 1 --2- 172.21.15.152:0/3479618295 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0248076c70 0x7f0248105440 unknown :-1 s=CLOSED pgs=1 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:00.868 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.231+0000 7f024ebfe700 1 -- 172.21.15.152:0/3479618295 >> 172.21.15.152:0/3479618295 conn(0x7f02480fcf30 msgr2=0x7f02480ff340 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:00.868 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.231+0000 7f024ebfe700 1 -- 172.21.15.152:0/3479618295 shutdown_connections 2024-04-02T04:49:00.868 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.231+0000 7f024ebfe700 1 -- 172.21.15.152:0/3479618295 wait complete. 2024-04-02T04:49:00.868 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.232+0000 7f024ebfe700 1 Processor -- start 2024-04-02T04:49:00.869 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.232+0000 7f024ebfe700 1 -- start start 2024-04-02T04:49:00.869 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.232+0000 7f024ebfe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0248076c70 0x7f0248110f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:00.869 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.232+0000 7f024ebfe700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0248111440 con 0x7f0248076c70 2024-04-02T04:49:00.869 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.232+0000 7f024c99a700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0248076c70 0x7f0248110f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:00.869 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.232+0000 7f024c99a700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0248076c70 0x7f0248110f00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:55434/0 (socket says 172.21.15.152:55434) 2024-04-02T04:49:00.869 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.232+0000 7f024c99a700 1 -- 172.21.15.152:0/1379524629 learned_addr learned my addr 172.21.15.152:0/1379524629 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:00.869 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.232+0000 7f024c99a700 1 -- 172.21.15.152:0/1379524629 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f023c0094f0 con 0x7f0248076c70 2024-04-02T04:49:00.869 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.233+0000 7f024c99a700 1 --2- 172.21.15.152:0/1379524629 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0248076c70 0x7f0248110f00 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7f023c009290 tx=0x7f023c016000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:00.869 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.233+0000 7f02457fa700 1 -- 172.21.15.152:0/1379524629 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f023c01c5c0 con 0x7f0248076c70 2024-04-02T04:49:00.869 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.233+0000 7f02457fa700 1 -- 172.21.15.152:0/1379524629 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f023c01c740 con 0x7f0248076c70 2024-04-02T04:49:00.869 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.233+0000 7f024ebfe700 1 -- 172.21.15.152:0/1379524629 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0248111640 con 0x7f0248076c70 2024-04-02T04:49:00.869 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.233+0000 7f02457fa700 1 -- 172.21.15.152:0/1379524629 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f023c01ca50 con 0x7f0248076c70 2024-04-02T04:49:00.869 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.233+0000 7f024ebfe700 1 -- 172.21.15.152:0/1379524629 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f02481120d0 con 0x7f0248076c70 2024-04-02T04:49:00.869 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.233+0000 7f02457fa700 1 -- 172.21.15.152:0/1379524629 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 1) v1 ==== 521+0+0 (secure 0 0 0) 0x7f023c01a450 con 0x7f0248076c70 2024-04-02T04:49:00.869 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.233+0000 7f02457fa700 1 -- 172.21.15.152:0/1379524629 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 721+0+0 (secure 0 0 0) 0x7f023c02e790 con 0x7f0248076c70 2024-04-02T04:49:00.869 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.234+0000 7f024ebfe700 1 -- 172.21.15.152:0/1379524629 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f024805f5f0 con 0x7f0248076c70 2024-04-02T04:49:00.870 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.236+0000 7f02457fa700 1 -- 172.21.15.152:0/1379524629 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+71349 (secure 0 0 0) 0x7f023c01ce30 con 0x7f0248076c70 2024-04-02T04:49:00.870 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.306+0000 7f024ebfe700 1 -- 172.21.15.152:0/1379524629 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "status"} v 0) v1 -- 0x7f0248105dd0 con 0x7f0248076c70 2024-04-02T04:49:00.870 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.306+0000 7f02457fa700 1 -- 172.21.15.152:0/1379524629 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "status"}]=0 v0) v1 ==== 54+0+325 (secure 0 0 0) 0x7f023c01a840 con 0x7f0248076c70 2024-04-02T04:49:00.870 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.307+0000 7f024ebfe700 1 -- 172.21.15.152:0/1379524629 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0248076c70 msgr2=0x7f0248110f00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:00.870 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.307+0000 7f024ebfe700 1 --2- 172.21.15.152:0/1379524629 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0248076c70 0x7f0248110f00 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7f023c009290 tx=0x7f023c016000 comp rx=0 tx=0).stop 2024-04-02T04:49:00.870 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.307+0000 7f024ebfe700 1 -- 172.21.15.152:0/1379524629 shutdown_connections 2024-04-02T04:49:00.870 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.307+0000 7f024ebfe700 1 --2- 172.21.15.152:0/1379524629 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0248076c70 0x7f0248110f00 unknown :-1 s=CLOSED pgs=2 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:00.870 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.307+0000 7f024ebfe700 1 -- 172.21.15.152:0/1379524629 >> 172.21.15.152:0/1379524629 conn(0x7f02480fcf30 msgr2=0x7f02480fd300 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:00.870 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.307+0000 7f024ebfe700 1 -- 172.21.15.152:0/1379524629 shutdown_connections 2024-04-02T04:49:00.870 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:00.307+0000 7f024ebfe700 1 -- 172.21.15.152:0/1379524629 wait complete. 2024-04-02T04:49:00.870 INFO:teuthology.orchestra.run.smithi152.stdout:mon is available 2024-04-02T04:49:00.870 INFO:teuthology.orchestra.run.smithi152.stdout:Assimilating anything we can from ceph.conf... 2024-04-02T04:49:01.006 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:00 smithi152 ceph-mon[28699]: mon.smithi152 is new leader, mons smithi152 in quorum (ranks 0) 2024-04-02T04:49:01.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:00 smithi152 ceph-mon[28699]: monmap e1: 1 mons at {smithi152=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-02T04:49:01.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:00 smithi152 ceph-mon[28699]: fsmap 2024-04-02T04:49:01.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:00 smithi152 ceph-mon[28699]: osdmap e1: 0 total, 0 up, 0 in 2024-04-02T04:49:01.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:00 smithi152 ceph-mon[28699]: mgrmap e1: no daemons active 2024-04-02T04:49:01.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:00 smithi152 ceph-mon[28699]: from='client.? 172.21.15.152:0/1379524629' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-02T04:49:01.949 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 2024-04-02T04:49:01.949 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout [global] 2024-04-02T04:49:01.949 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout fsid = 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 2024-04-02T04:49:01.949 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.152:3300,v1:172.21.15.152:6789] 2024-04-02T04:49:01.949 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-02T04:49:01.949 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-02T04:49:01.949 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-02T04:49:01.949 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-02T04:49:01.949 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 2024-04-02T04:49:01.949 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-02T04:49:01.949 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-02T04:49:01.950 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 2024-04-02T04:49:01.950 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout [osd] 2024-04-02T04:49:01.950 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mon_osd_backfillfull_ratio = 0.85 2024-04-02T04:49:01.950 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mon_osd_full_ratio = 0.9 2024-04-02T04:49:01.950 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mon_osd_nearfull_ratio = 0.8 2024-04-02T04:49:01.950 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-02T04:49:01.950 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-02T04:49:01.950 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.336+0000 7fcc844a1700 1 Processor -- start 2024-04-02T04:49:01.950 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.336+0000 7fcc844a1700 1 -- start start 2024-04-02T04:49:01.950 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.337+0000 7fcc844a1700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcc7c1028c0 0x7fcc7c102cd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:01.950 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.337+0000 7fcc844a1700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcc7c103210 con 0x7fcc7c1028c0 2024-04-02T04:49:01.950 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.337+0000 7fcc8223d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcc7c1028c0 0x7fcc7c102cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:01.950 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.337+0000 7fcc8223d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcc7c1028c0 0x7fcc7c102cd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:43276/0 (socket says 172.21.15.152:43276) 2024-04-02T04:49:01.950 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.337+0000 7fcc8223d700 1 -- 172.21.15.152:0/1645426865 learned_addr learned my addr 172.21.15.152:0/1645426865 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:01.951 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.337+0000 7fcc8223d700 1 -- 172.21.15.152:0/1645426865 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcc7c103350 con 0x7fcc7c1028c0 2024-04-02T04:49:01.951 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.337+0000 7fcc8223d700 1 --2- 172.21.15.152:0/1645426865 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcc7c1028c0 0x7fcc7c102cd0 secure :-1 s=READY pgs=3 cs=0 l=1 rev1=1 crypto rx=0x7fcc740098e0 tx=0x7fcc74009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a3b62d298bc7ff9d server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:01.951 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.338+0000 7fcc8123b700 1 -- 172.21.15.152:0/1645426865 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fcc7400b860 con 0x7fcc7c1028c0 2024-04-02T04:49:01.951 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.338+0000 7fcc8123b700 1 -- 172.21.15.152:0/1645426865 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcc7400b9c0 con 0x7fcc7c1028c0 2024-04-02T04:49:01.951 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.338+0000 7fcc8123b700 1 -- 172.21.15.152:0/1645426865 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fcc7400bc90 con 0x7fcc7c1028c0 2024-04-02T04:49:01.951 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.338+0000 7fcc844a1700 1 -- 172.21.15.152:0/1645426865 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcc7c1028c0 msgr2=0x7fcc7c102cd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:01.951 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.338+0000 7fcc844a1700 1 --2- 172.21.15.152:0/1645426865 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcc7c1028c0 0x7fcc7c102cd0 secure :-1 s=READY pgs=3 cs=0 l=1 rev1=1 crypto rx=0x7fcc740098e0 tx=0x7fcc74009250 comp rx=0 tx=0).stop 2024-04-02T04:49:01.951 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.338+0000 7fcc844a1700 1 -- 172.21.15.152:0/1645426865 shutdown_connections 2024-04-02T04:49:01.951 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.338+0000 7fcc844a1700 1 --2- 172.21.15.152:0/1645426865 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcc7c1028c0 0x7fcc7c102cd0 unknown :-1 s=CLOSED pgs=3 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:01.951 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.338+0000 7fcc844a1700 1 -- 172.21.15.152:0/1645426865 >> 172.21.15.152:0/1645426865 conn(0x7fcc7c0fe040 msgr2=0x7fcc7c100450 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:01.951 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.338+0000 7fcc844a1700 1 -- 172.21.15.152:0/1645426865 shutdown_connections 2024-04-02T04:49:01.951 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.338+0000 7fcc844a1700 1 -- 172.21.15.152:0/1645426865 wait complete. 2024-04-02T04:49:01.951 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.339+0000 7fcc844a1700 1 Processor -- start 2024-04-02T04:49:01.951 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.339+0000 7fcc844a1700 1 -- start start 2024-04-02T04:49:01.951 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.339+0000 7fcc844a1700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcc7c1028c0 0x7fcc7c110f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:01.951 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.339+0000 7fcc844a1700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcc7c111460 con 0x7fcc7c1028c0 2024-04-02T04:49:01.952 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.339+0000 7fcc8223d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcc7c1028c0 0x7fcc7c110f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:01.952 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.339+0000 7fcc8223d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcc7c1028c0 0x7fcc7c110f20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:43282/0 (socket says 172.21.15.152:43282) 2024-04-02T04:49:01.952 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.339+0000 7fcc8223d700 1 -- 172.21.15.152:0/4124532737 learned_addr learned my addr 172.21.15.152:0/4124532737 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:01.952 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.339+0000 7fcc8223d700 1 -- 172.21.15.152:0/4124532737 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcc740094f0 con 0x7fcc7c1028c0 2024-04-02T04:49:01.952 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.339+0000 7fcc8223d700 1 --2- 172.21.15.152:0/4124532737 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcc7c1028c0 0x7fcc7c110f20 secure :-1 s=READY pgs=4 cs=0 l=1 rev1=1 crypto rx=0x7fcc74003080 tx=0x7fcc74016000 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:01.952 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.339+0000 7fcc7affd700 1 -- 172.21.15.152:0/4124532737 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fcc7401c5c0 con 0x7fcc7c1028c0 2024-04-02T04:49:01.952 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.339+0000 7fcc7affd700 1 -- 172.21.15.152:0/4124532737 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7fcc7401c720 con 0x7fcc7c1028c0 2024-04-02T04:49:01.952 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.340+0000 7fcc844a1700 1 -- 172.21.15.152:0/4124532737 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcc7c111660 con 0x7fcc7c1028c0 2024-04-02T04:49:01.952 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.340+0000 7fcc7affd700 1 -- 172.21.15.152:0/4124532737 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fcc7401ca30 con 0x7fcc7c1028c0 2024-04-02T04:49:01.952 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.340+0000 7fcc844a1700 1 -- 172.21.15.152:0/4124532737 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcc7c112090 con 0x7fcc7c1028c0 2024-04-02T04:49:01.952 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.340+0000 7fcc7affd700 1 -- 172.21.15.152:0/4124532737 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 1) v1 ==== 521+0+0 (secure 0 0 0) 0x7fcc7401a450 con 0x7fcc7c1028c0 2024-04-02T04:49:01.952 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.340+0000 7fcc7affd700 1 -- 172.21.15.152:0/4124532737 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 721+0+0 (secure 0 0 0) 0x7fcc7402e790 con 0x7fcc7c1028c0 2024-04-02T04:49:01.952 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.341+0000 7fcc844a1700 1 -- 172.21.15.152:0/4124532737 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcc7c05f5f0 con 0x7fcc7c1028c0 2024-04-02T04:49:01.952 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.343+0000 7fcc7affd700 1 -- 172.21.15.152:0/4124532737 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+71349 (secure 0 0 0) 0x7fcc7401a890 con 0x7fcc7c1028c0 2024-04-02T04:49:01.952 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.409+0000 7fcc844a1700 1 -- 172.21.15.152:0/4124532737 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "config assimilate-conf"} v 0) v1 -- 0x7fcc7c10abb0 con 0x7fcc7c1028c0 2024-04-02T04:49:01.952 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.413+0000 7fcc7affd700 1 -- 172.21.15.152:0/4124532737 <== mon.0 v2:172.21.15.152:3300/0 7 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7fcc7401aa30 con 0x7fcc7c1028c0 2024-04-02T04:49:01.953 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.414+0000 7fcc7affd700 1 -- 172.21.15.152:0/4124532737 <== mon.0 v2:172.21.15.152:3300/0 8 ==== mon_command_ack([{"prefix": "config assimilate-conf"}]=0 v2) v1 ==== 70+0+431 (secure 0 0 0) 0x7fcc74024910 con 0x7fcc7c1028c0 2024-04-02T04:49:01.953 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.414+0000 7fcc844a1700 1 -- 172.21.15.152:0/4124532737 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcc7c1028c0 msgr2=0x7fcc7c110f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:01.953 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.414+0000 7fcc844a1700 1 --2- 172.21.15.152:0/4124532737 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcc7c1028c0 0x7fcc7c110f20 secure :-1 s=READY pgs=4 cs=0 l=1 rev1=1 crypto rx=0x7fcc74003080 tx=0x7fcc74016000 comp rx=0 tx=0).stop 2024-04-02T04:49:01.953 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.414+0000 7fcc844a1700 1 -- 172.21.15.152:0/4124532737 shutdown_connections 2024-04-02T04:49:01.953 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.414+0000 7fcc844a1700 1 --2- 172.21.15.152:0/4124532737 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcc7c1028c0 0x7fcc7c110f20 unknown :-1 s=CLOSED pgs=4 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:01.953 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.414+0000 7fcc844a1700 1 -- 172.21.15.152:0/4124532737 >> 172.21.15.152:0/4124532737 conn(0x7fcc7c0fe040 msgr2=0x7fcc7c107d30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:01.953 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.415+0000 7fcc844a1700 1 -- 172.21.15.152:0/4124532737 shutdown_connections 2024-04-02T04:49:01.953 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:01.415+0000 7fcc844a1700 1 -- 172.21.15.152:0/4124532737 wait complete. 2024-04-02T04:49:01.953 INFO:teuthology.orchestra.run.smithi152.stdout:Generating new minimal ceph.conf... 2024-04-02T04:49:02.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:01 smithi152 ceph-mon[28699]: from='client.? 172.21.15.152:0/4124532737' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-02T04:49:02.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:01 smithi152 ceph-mon[28699]: from='client.? 172.21.15.152:0/4124532737' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-02T04:49:02.954 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.349+0000 7f1884ae9700 1 Processor -- start 2024-04-02T04:49:02.954 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.349+0000 7f1884ae9700 1 -- start start 2024-04-02T04:49:02.954 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.349+0000 7f1884ae9700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f1880100ce0 0x7f1880101110 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:02.954 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.349+0000 7f1884ae9700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1880103b40 con 0x7f1880100ce0 2024-04-02T04:49:02.954 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.349+0000 7f187e59c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f1880100ce0 0x7f1880101110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:02.954 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.349+0000 7f187e59c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f1880100ce0 0x7f1880101110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:43284/0 (socket says 172.21.15.152:43284) 2024-04-02T04:49:02.954 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.349+0000 7f187e59c700 1 -- 172.21.15.152:0/1817341291 learned_addr learned my addr 172.21.15.152:0/1817341291 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:02.954 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.350+0000 7f187e59c700 1 -- 172.21.15.152:0/1817341291 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1880101650 con 0x7f1880100ce0 2024-04-02T04:49:02.955 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.350+0000 7f187e59c700 1 --2- 172.21.15.152:0/1817341291 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f1880100ce0 0x7f1880101110 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7f18700098e0 tx=0x7f1870009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c8acdd52f73cd153 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:02.955 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.350+0000 7f187dd9b700 1 -- 172.21.15.152:0/1817341291 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f187000b860 con 0x7f1880100ce0 2024-04-02T04:49:02.955 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.350+0000 7f187dd9b700 1 -- 172.21.15.152:0/1817341291 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7f187000be50 con 0x7f1880100ce0 2024-04-02T04:49:02.955 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.350+0000 7f187dd9b700 1 -- 172.21.15.152:0/1817341291 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1870010380 con 0x7f1880100ce0 2024-04-02T04:49:02.955 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.351+0000 7f1884ae9700 1 -- 172.21.15.152:0/1817341291 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f1880100ce0 msgr2=0x7f1880101110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:02.955 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.351+0000 7f1884ae9700 1 --2- 172.21.15.152:0/1817341291 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f1880100ce0 0x7f1880101110 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7f18700098e0 tx=0x7f1870009250 comp rx=0 tx=0).stop 2024-04-02T04:49:02.955 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.351+0000 7f1884ae9700 1 -- 172.21.15.152:0/1817341291 shutdown_connections 2024-04-02T04:49:02.955 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.351+0000 7f1884ae9700 1 --2- 172.21.15.152:0/1817341291 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f1880100ce0 0x7f1880101110 unknown :-1 s=CLOSED pgs=5 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:02.955 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.351+0000 7f1884ae9700 1 -- 172.21.15.152:0/1817341291 >> 172.21.15.152:0/1817341291 conn(0x7f18800fc6e0 msgr2=0x7f18800feb10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:02.955 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.351+0000 7f1884ae9700 1 -- 172.21.15.152:0/1817341291 shutdown_connections 2024-04-02T04:49:02.955 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.351+0000 7f1884ae9700 1 -- 172.21.15.152:0/1817341291 wait complete. 2024-04-02T04:49:02.955 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.351+0000 7f1884ae9700 1 Processor -- start 2024-04-02T04:49:02.955 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.351+0000 7f1884ae9700 1 -- start start 2024-04-02T04:49:02.955 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.352+0000 7f1884ae9700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f1880100ce0 0x7f18801138e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:02.955 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.352+0000 7f1884ae9700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1880113e20 con 0x7f1880100ce0 2024-04-02T04:49:02.956 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.352+0000 7f187e59c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f1880100ce0 0x7f18801138e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:02.956 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.352+0000 7f187e59c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f1880100ce0 0x7f18801138e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:43294/0 (socket says 172.21.15.152:43294) 2024-04-02T04:49:02.956 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.352+0000 7f187e59c700 1 -- 172.21.15.152:0/361506159 learned_addr learned my addr 172.21.15.152:0/361506159 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:02.956 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.352+0000 7f187e59c700 1 -- 172.21.15.152:0/361506159 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f18700094f0 con 0x7f1880100ce0 2024-04-02T04:49:02.956 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.352+0000 7f187e59c700 1 --2- 172.21.15.152:0/361506159 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f1880100ce0 0x7f18801138e0 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7f1870003050 tx=0x7f1870009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:02.956 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.352+0000 7f18777fe700 1 -- 172.21.15.152:0/361506159 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1870010760 con 0x7f1880100ce0 2024-04-02T04:49:02.956 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.352+0000 7f18777fe700 1 -- 172.21.15.152:0/361506159 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7f1870010d50 con 0x7f1880100ce0 2024-04-02T04:49:02.956 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.352+0000 7f1884ae9700 1 -- 172.21.15.152:0/361506159 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1880114020 con 0x7f1880100ce0 2024-04-02T04:49:02.956 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.352+0000 7f18777fe700 1 -- 172.21.15.152:0/361506159 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f187001ae20 con 0x7f1880100ce0 2024-04-02T04:49:02.956 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.352+0000 7f1884ae9700 1 -- 172.21.15.152:0/361506159 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1880114b90 con 0x7f1880100ce0 2024-04-02T04:49:02.956 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.352+0000 7f18777fe700 1 -- 172.21.15.152:0/361506159 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 1) v1 ==== 521+0+0 (secure 0 0 0) 0x7f187002e6c0 con 0x7f1880100ce0 2024-04-02T04:49:02.956 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.352+0000 7f18777fe700 1 -- 172.21.15.152:0/361506159 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 721+0+0 (secure 0 0 0) 0x7f1870024860 con 0x7f1880100ce0 2024-04-02T04:49:02.956 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.353+0000 7f1884ae9700 1 -- 172.21.15.152:0/361506159 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f188005f5f0 con 0x7f1880100ce0 2024-04-02T04:49:02.956 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.355+0000 7f18777fe700 1 -- 172.21.15.152:0/361506159 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+71349 (secure 0 0 0) 0x7f1870029070 con 0x7f1880100ce0 2024-04-02T04:49:02.957 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.421+0000 7f1884ae9700 1 -- 172.21.15.152:0/361506159 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "config generate-minimal-conf"} v 0) v1 -- 0x7f1880114e90 con 0x7f1880100ce0 2024-04-02T04:49:02.957 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.421+0000 7f18777fe700 1 -- 172.21.15.152:0/361506159 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "config generate-minimal-conf"}]=0 v2) v1 ==== 76+0+177 (secure 0 0 0) 0x7f187001a6a0 con 0x7f1880100ce0 2024-04-02T04:49:02.957 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.422+0000 7f1884ae9700 1 -- 172.21.15.152:0/361506159 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f1880100ce0 msgr2=0x7f18801138e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:02.957 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.422+0000 7f1884ae9700 1 --2- 172.21.15.152:0/361506159 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f1880100ce0 0x7f18801138e0 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7f1870003050 tx=0x7f1870009250 comp rx=0 tx=0).stop 2024-04-02T04:49:02.957 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.422+0000 7f1884ae9700 1 -- 172.21.15.152:0/361506159 shutdown_connections 2024-04-02T04:49:02.957 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.422+0000 7f1884ae9700 1 --2- 172.21.15.152:0/361506159 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f1880100ce0 0x7f18801138e0 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:02.957 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.422+0000 7f1884ae9700 1 -- 172.21.15.152:0/361506159 >> 172.21.15.152:0/361506159 conn(0x7f18800fc6e0 msgr2=0x7f18800fd320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:02.957 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.422+0000 7f1884ae9700 1 -- 172.21.15.152:0/361506159 shutdown_connections 2024-04-02T04:49:02.957 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:02.422+0000 7f1884ae9700 1 -- 172.21.15.152:0/361506159 wait complete. 2024-04-02T04:49:02.957 INFO:teuthology.orchestra.run.smithi152.stdout:Restarting the monitor... 2024-04-02T04:49:03.006 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:02 smithi152 ceph-mon[28699]: from='client.? 172.21.15.152:0/361506159' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:49:03.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:02 smithi152 systemd[1]: Stopping Ceph mon.smithi152 for 2e0055d0-f0ac-11ee-b647-cb9ed24678a4... 2024-04-02T04:49:03.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:03 smithi152 ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-mon-smithi152[28695]: 2024-04-02T04:49:03.188+0000 7f1b911c8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi152 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-02T04:49:03.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:03 smithi152 ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-mon-smithi152[28695]: 2024-04-02T04:49:03.188+0000 7f1b911c8700 -1 mon.smithi152@0(leader) e1 *** Got Signal Terminated *** 2024-04-02T04:49:03.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:03 smithi152 podman[28944]: 2024-04-02 04:49:03.434740798 +0000 UTC m=+0.325649895 container died 334f706d66e0a0611e54df430be3c7b875dcb026e3c798c242f63bebb90b7288 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-mon-smithi152, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, RELEASE=quincy-c924f99, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-02T04:49:03.942 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:03 smithi152 podman[28944]: 2024-04-02 04:49:03.747389432 +0000 UTC m=+0.638298515 container cleanup 334f706d66e0a0611e54df430be3c7b875dcb026e3c798c242f63bebb90b7288 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-mon-smithi152, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=quincy-c924f99, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-04-02T04:49:03.943 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:03 smithi152 bash[28944]: ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-mon-smithi152 2024-04-02T04:49:04.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:03 smithi152 podman[28958]: 2024-04-02 04:49:03.940005966 +0000 UTC m=+0.502368621 container remove 334f706d66e0a0611e54df430be3c7b875dcb026e3c798c242f63bebb90b7288 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-mon-smithi152, org.label-schema.vendor=CentOS, RELEASE=quincy-c924f99, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-02T04:49:04.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 systemd[1]: ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4@mon.smithi152.service: Deactivated successfully. 2024-04-02T04:49:04.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 systemd[1]: Stopped Ceph mon.smithi152 for 2e0055d0-f0ac-11ee-b647-cb9ed24678a4. 2024-04-02T04:49:04.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 systemd[1]: Starting Ceph mon.smithi152 for 2e0055d0-f0ac-11ee-b647-cb9ed24678a4... 2024-04-02T04:49:04.622 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 podman[29054]: 2024-04-02 04:49:04.358031044 +0000 UTC m=+0.100924716 container create 399b9cc778f0e8f17fbe930b7447197a7efa8e2abbd1a1ac20f7143cbba296e5 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-mon-smithi152, CEPH_POINT_RELEASE=, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=quincy-c924f99, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-02T04:49:04.622 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 podman[29054]: 2024-04-02 04:49:04.27482125 +0000 UTC m=+0.017714942 image pull 4948510fac0f2ac7df4b4e361573714a48c110d48535a4b8d5aeb81e8023bedd quay.ceph.io/ceph-ci/ceph:quincy 2024-04-02T04:49:04.622 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 podman[29054]: 2024-04-02 04:49:04.543654285 +0000 UTC m=+0.286547962 container init 399b9cc778f0e8f17fbe930b7447197a7efa8e2abbd1a1ac20f7143cbba296e5 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-mon-smithi152, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, RELEASE=quincy-c924f99, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_CLEAN=True) 2024-04-02T04:49:04.622 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 podman[29054]: 2024-04-02 04:49:04.546501017 +0000 UTC m=+0.289394689 container start 399b9cc778f0e8f17fbe930b7447197a7efa8e2abbd1a1ac20f7143cbba296e5 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-mon-smithi152, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=quincy-c924f99, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-04-02T04:49:04.622 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: set uid:gid to 167:167 (ceph:ceph) 2024-04-02T04:49:04.623 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: ceph version 17.2.7-803-gc924f991 (c924f9914f2809997b363f77c5cbb14929881220) quincy (stable), process ceph-mon, pid 2 2024-04-02T04:49:04.623 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: pidfile_write: ignore empty --pid-file 2024-04-02T04:49:04.623 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: load: jerasure load: lrc 2024-04-02T04:49:04.623 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: RocksDB version: 6.15.5 2024-04-02T04:49:04.623 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-02T04:49:04.623 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Compile date Apr 1 2024 2024-04-02T04:49:04.623 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: DB SUMMARY 2024-04-02T04:49:04.623 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: DB Session ID: 20MRBSLLH0N57CUN0DPF 2024-04-02T04:49:04.623 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: CURRENT file: CURRENT 2024-04-02T04:49:04.623 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: IDENTITY file: IDENTITY 2024-04-02T04:49:04.623 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-02T04:49:04.623 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi152/store.db dir, Total Num: 1, files: 000008.sst 2024-04-02T04:49:04.623 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi152/store.db: 000010.log size: 98174 ; 2024-04-02T04:49:04.624 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.error_if_exists: 0 2024-04-02T04:49:04.624 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.create_if_missing: 0 2024-04-02T04:49:04.624 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.paranoid_checks: 1 2024-04-02T04:49:04.624 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-02T04:49:04.624 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.env: 0x5631edda2a80 2024-04-02T04:49:04.624 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.fs: Posix File System 2024-04-02T04:49:04.624 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.info_log: 0x5631eeeebc00 2024-04-02T04:49:04.624 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_file_opening_threads: 16 2024-04-02T04:49:04.624 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.statistics: (nil) 2024-04-02T04:49:04.624 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.use_fsync: 0 2024-04-02T04:49:04.624 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_log_file_size: 0 2024-04-02T04:49:04.624 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-02T04:49:04.624 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-02T04:49:04.624 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.keep_log_file_num: 1000 2024-04-02T04:49:04.625 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.recycle_log_file_num: 0 2024-04-02T04:49:04.625 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.allow_fallocate: 1 2024-04-02T04:49:04.625 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.allow_mmap_reads: 0 2024-04-02T04:49:04.625 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.allow_mmap_writes: 0 2024-04-02T04:49:04.625 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.use_direct_reads: 0 2024-04-02T04:49:04.625 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-02T04:49:04.625 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.create_missing_column_families: 0 2024-04-02T04:49:04.625 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.db_log_dir: 2024-04-02T04:49:04.625 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi152/store.db 2024-04-02T04:49:04.625 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-02T04:49:04.625 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-02T04:49:04.625 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-02T04:49:04.625 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-02T04:49:04.625 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-02T04:49:04.625 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-02T04:49:04.626 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.advise_random_on_open: 1 2024-04-02T04:49:04.626 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.db_write_buffer_size: 0 2024-04-02T04:49:04.626 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.write_buffer_manager: 0x5631efc60990 2024-04-02T04:49:04.626 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-02T04:49:04.626 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-02T04:49:04.626 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-02T04:49:04.626 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-02T04:49:04.626 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.rate_limiter: (nil) 2024-04-02T04:49:04.626 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-02T04:49:04.626 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.wal_recovery_mode: 2 2024-04-02T04:49:04.626 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.enable_thread_tracking: 0 2024-04-02T04:49:04.626 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.enable_pipelined_write: 0 2024-04-02T04:49:04.626 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.unordered_write: 0 2024-04-02T04:49:04.627 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-02T04:49:04.627 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-02T04:49:04.627 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-02T04:49:04.627 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-02T04:49:04.627 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.row_cache: None 2024-04-02T04:49:04.627 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.wal_filter: None 2024-04-02T04:49:04.627 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-02T04:49:04.627 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.allow_ingest_behind: 0 2024-04-02T04:49:04.627 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.preserve_deletes: 0 2024-04-02T04:49:04.627 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.two_write_queues: 0 2024-04-02T04:49:04.627 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.manual_wal_flush: 0 2024-04-02T04:49:04.627 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.atomic_flush: 0 2024-04-02T04:49:04.627 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-02T04:49:04.627 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-02T04:49:04.627 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-02T04:49:04.627 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.log_readahead_size: 0 2024-04-02T04:49:04.628 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-02T04:49:04.628 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.best_efforts_recovery: 0 2024-04-02T04:49:04.628 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-02T04:49:04.628 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-02T04:49:04.628 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.allow_data_in_errors: 0 2024-04-02T04:49:04.628 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.db_host_id: __hostname__ 2024-04-02T04:49:04.628 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_background_jobs: 2 2024-04-02T04:49:04.628 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_background_compactions: -1 2024-04-02T04:49:04.628 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_subcompactions: 1 2024-04-02T04:49:04.628 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-02T04:49:04.628 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-02T04:49:04.628 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-02T04:49:04.628 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_total_wal_size: 0 2024-04-02T04:49:04.628 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-02T04:49:04.628 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-02T04:49:04.629 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-02T04:49:04.629 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-02T04:49:04.629 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_open_files: -1 2024-04-02T04:49:04.629 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.bytes_per_sync: 0 2024-04-02T04:49:04.629 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-02T04:49:04.629 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-02T04:49:04.629 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.compaction_readahead_size: 0 2024-04-02T04:49:04.629 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_background_flushes: -1 2024-04-02T04:49:04.630 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Compression algorithms supported: 2024-04-02T04:49:04.630 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-02T04:49:04.630 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: kZSTD supported: 0 2024-04-02T04:49:04.630 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: kXpressCompression supported: 0 2024-04-02T04:49:04.630 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: kLZ4HCCompression supported: 1 2024-04-02T04:49:04.630 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: kZlibCompression supported: 1 2024-04-02T04:49:04.631 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: kSnappyCompression supported: 1 2024-04-02T04:49:04.631 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: kLZ4Compression supported: 1 2024-04-02T04:49:04.631 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: kBZip2Compression supported: 0 2024-04-02T04:49:04.631 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-02T04:49:04.631 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi152/store.db/MANIFEST-000009 2024-04-02T04:49:04.631 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-04-02T04:49:04.631 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-02T04:49:04.631 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.merge_operator: 2024-04-02T04:49:04.631 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.compaction_filter: None 2024-04-02T04:49:04.631 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.compaction_filter_factory: None 2024-04-02T04:49:04.631 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.sst_partitioner_factory: None 2024-04-02T04:49:04.631 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-02T04:49:04.631 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-02T04:49:04.631 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5631eef40f58) 2024-04-02T04:49:04.632 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: cache_index_and_filter_blocks: 1 2024-04-02T04:49:04.632 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-02T04:49:04.632 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-02T04:49:04.632 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: pin_top_level_index_and_filter: 1 2024-04-02T04:49:04.632 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: index_type: 0 2024-04-02T04:49:04.632 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: data_block_index_type: 0 2024-04-02T04:49:04.632 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: index_shortening: 1 2024-04-02T04:49:04.632 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-02T04:49:04.632 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: hash_index_allow_collision: 1 2024-04-02T04:49:04.632 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: checksum: 1 2024-04-02T04:49:04.632 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: no_block_cache: 0 2024-04-02T04:49:04.632 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: block_cache: 0x5631eef73090 2024-04-02T04:49:04.632 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: block_cache_name: BinnedLRUCache 2024-04-02T04:49:04.632 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: block_cache_options: 2024-04-02T04:49:04.632 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: capacity : 536870912 2024-04-02T04:49:04.632 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: num_shard_bits : 4 2024-04-02T04:49:04.633 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: strict_capacity_limit : 0 2024-04-02T04:49:04.633 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: high_pri_pool_ratio: 0.000 2024-04-02T04:49:04.633 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: block_cache_compressed: (nil) 2024-04-02T04:49:04.633 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: persistent_cache: (nil) 2024-04-02T04:49:04.633 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: block_size: 4096 2024-04-02T04:49:04.633 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: block_size_deviation: 10 2024-04-02T04:49:04.633 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: block_restart_interval: 16 2024-04-02T04:49:04.633 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: index_block_restart_interval: 1 2024-04-02T04:49:04.633 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: metadata_block_size: 4096 2024-04-02T04:49:04.633 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: partition_filters: 0 2024-04-02T04:49:04.633 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: use_delta_encoding: 1 2024-04-02T04:49:04.633 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-02T04:49:04.633 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: whole_key_filtering: 1 2024-04-02T04:49:04.633 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: verify_compression: 0 2024-04-02T04:49:04.633 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: read_amp_bytes_per_bit: 0 2024-04-02T04:49:04.633 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: format_version: 4 2024-04-02T04:49:04.634 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: enable_index_compression: 1 2024-04-02T04:49:04.634 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: block_align: 0 2024-04-02T04:49:04.634 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.write_buffer_size: 33554432 2024-04-02T04:49:04.634 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_write_buffer_number: 2 2024-04-02T04:49:04.634 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.compression: NoCompression 2024-04-02T04:49:04.634 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.bottommost_compression: Disabled 2024-04-02T04:49:04.634 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.prefix_extractor: nullptr 2024-04-02T04:49:04.634 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-02T04:49:04.634 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.num_levels: 7 2024-04-02T04:49:04.634 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-02T04:49:04.634 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-02T04:49:04.634 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-02T04:49:04.634 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-02T04:49:04.635 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-02T04:49:04.635 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-02T04:49:04.635 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-02T04:49:04.635 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-02T04:49:04.635 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-02T04:49:04.635 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-02T04:49:04.635 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-02T04:49:04.635 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.compression_opts.level: 32767 2024-04-02T04:49:04.635 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.compression_opts.strategy: 0 2024-04-02T04:49:04.635 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-02T04:49:04.635 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-02T04:49:04.635 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-02T04:49:04.635 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.compression_opts.enabled: false 2024-04-02T04:49:04.635 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-02T04:49:04.635 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-02T04:49:04.636 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-02T04:49:04.636 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.target_file_size_base: 67108864 2024-04-02T04:49:04.636 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-02T04:49:04.636 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-02T04:49:04.636 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-02T04:49:04.636 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-02T04:49:04.636 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-02T04:49:04.636 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-02T04:49:04.636 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-02T04:49:04.636 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-02T04:49:04.636 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-02T04:49:04.636 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-02T04:49:04.636 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-02T04:49:04.636 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-02T04:49:04.636 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-02T04:49:04.637 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.arena_block_size: 4194304 2024-04-02T04:49:04.637 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-02T04:49:04.637 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-02T04:49:04.637 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-02T04:49:04.637 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.disable_auto_compactions: 0 2024-04-02T04:49:04.637 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-02T04:49:04.637 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-02T04:49:04.637 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-02T04:49:04.637 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-02T04:49:04.637 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-02T04:49:04.637 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-02T04:49:04.637 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-02T04:49:04.637 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-02T04:49:04.637 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-02T04:49:04.637 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-02T04:49:04.637 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.table_properties_collectors: 2024-04-02T04:49:04.638 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.inplace_update_support: 0 2024-04-02T04:49:04.638 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-02T04:49:04.638 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-02T04:49:04.638 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-02T04:49:04.638 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-02T04:49:04.638 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.bloom_locality: 0 2024-04-02T04:49:04.638 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.max_successive_merges: 0 2024-04-02T04:49:04.638 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-02T04:49:04.638 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.paranoid_file_checks: 0 2024-04-02T04:49:04.638 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.force_consistency_checks: 1 2024-04-02T04:49:04.638 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.report_bg_io_stats: 0 2024-04-02T04:49:04.638 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.ttl: 2592000 2024-04-02T04:49:04.638 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-02T04:49:04.639 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.enable_blob_files: false 2024-04-02T04:49:04.639 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.min_blob_size: 0 2024-04-02T04:49:04.639 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.blob_file_size: 268435456 2024-04-02T04:49:04.639 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-02T04:49:04.639 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-02T04:49:04.639 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-02T04:49:04.639 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi152/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-02T04:49:04.639 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-04-02T04:49:04.639 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-04-02T04:49:04.639 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712033344572415, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-02T04:49:04.639 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-04-02T04:49:04.639 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-02T04:49:04.639 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712033344573301, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 93380, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 91492, "index_size": 248, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 709, "raw_key_size": 13576, "raw_average_key_size": 49, "raw_value_size": 84056, "raw_average_value_size": 307, "num_data_blocks": 11, "num_entries": 273, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1712033344, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "5cf67722-8f7a-489c-bde1-bc71c7f3cf7b", "db_session_id": "20MRBSLLH0N57CUN0DPF"}} 2024-04-02T04:49:04.639 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-04-02T04:49:04.639 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712033344573650, "job": 1, "event": "recovery_finished"} 2024-04-02T04:49:04.640 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-smithi152/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-02T04:49:04.640 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5631eef07180 2024-04-02T04:49:04.640 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: DB pointer 0x5631eefdc000 2024-04-02T04:49:04.640 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-04-02T04:49:04.640 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-02T04:49:04.640 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: ** DB Stats ** 2024-04-02T04:49:04.640 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T04:49:04.640 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-02T04:49:04.640 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-02T04:49:04.640 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-02T04:49:04.640 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-02T04:49:04.640 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-02T04:49:04.640 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-02T04:49:04.640 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: 2024-04-02T04:49:04.640 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: ** Compaction Stats [default] ** 2024-04-02T04:49:04.640 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T04:49:04.641 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T04:49:04.641 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: L0 2/0 92.99 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 142.7 0.00 0.00 1 0.001 0 0 2024-04-02T04:49:04.641 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: Sum 2/0 92.99 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 142.7 0.00 0.00 1 0.001 0 0 2024-04-02T04:49:04.641 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 142.7 0.00 0.00 1 0.001 0 0 2024-04-02T04:49:04.641 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: 2024-04-02T04:49:04.641 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: ** Compaction Stats [default] ** 2024-04-02T04:49:04.641 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T04:49:04.641 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T04:49:04.641 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 142.7 0.00 0.00 1 0.001 0 0 2024-04-02T04:49:04.641 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T04:49:04.641 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-02T04:49:04.641 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-02T04:49:04.641 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-02T04:49:04.641 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-02T04:49:04.642 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-02T04:49:04.642 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: Cumulative compaction: 0.00 GB write, 28.97 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T04:49:04.642 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: Interval compaction: 0.00 GB write, 28.97 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T04:49:04.642 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-02T04:49:04.642 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: 2024-04-02T04:49:04.642 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-02T04:49:04.642 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: 2024-04-02T04:49:04.642 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: ** Compaction Stats [default] ** 2024-04-02T04:49:04.642 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T04:49:04.642 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T04:49:04.642 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: L0 2/0 92.99 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 142.7 0.00 0.00 1 0.001 0 0 2024-04-02T04:49:04.642 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: Sum 2/0 92.99 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 142.7 0.00 0.00 1 0.001 0 0 2024-04-02T04:49:04.642 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-02T04:49:04.642 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: 2024-04-02T04:49:04.643 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: ** Compaction Stats [default] ** 2024-04-02T04:49:04.643 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T04:49:04.643 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T04:49:04.643 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 142.7 0.00 0.00 1 0.001 0 0 2024-04-02T04:49:04.643 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T04:49:04.643 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-02T04:49:04.643 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-02T04:49:04.643 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-02T04:49:04.643 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-02T04:49:04.643 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-02T04:49:04.643 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: Cumulative compaction: 0.00 GB write, 28.69 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T04:49:04.643 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T04:49:04.643 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-02T04:49:04.643 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: 2024-04-02T04:49:04.644 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-02T04:49:04.644 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: starting mon.smithi152 rank 0 at public addrs [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] at bind addrs [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi152 fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 2024-04-02T04:49:04.644 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: mon.smithi152@-1(???) e1 preinit fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 2024-04-02T04:49:04.644 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: mon.smithi152@-1(???).mds e1 new map 2024-04-02T04:49:04.644 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: mon.smithi152@-1(???).mds e1 print_map 2024-04-02T04:49:04.644 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: e1 2024-04-02T04:49:04.644 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-02T04:49:04.644 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-02T04:49:04.644 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: legacy client fscid: -1 2024-04-02T04:49:04.644 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: 2024-04-02T04:49:04.644 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout: No filesystems configured 2024-04-02T04:49:04.644 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: mon.smithi152@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-02T04:49:04.644 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: mon.smithi152@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T04:49:04.644 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: mon.smithi152@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T04:49:04.644 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: mon.smithi152@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T04:49:04.644 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: mon.smithi152@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-02T04:49:04.645 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 bash[29054]: 399b9cc778f0e8f17fbe930b7447197a7efa8e2abbd1a1ac20f7143cbba296e5 2024-04-02T04:49:04.645 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 systemd[1]: Started Ceph mon.smithi152 for 2e0055d0-f0ac-11ee-b647-cb9ed24678a4. 2024-04-02T04:49:04.645 INFO:teuthology.orchestra.run.smithi152.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-04-02T04:49:04.966 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: mon.smithi152 is new leader, mons smithi152 in quorum (ranks 0) 2024-04-02T04:49:04.967 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: monmap e1: 1 mons at {smithi152=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-02T04:49:04.967 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: fsmap 2024-04-02T04:49:04.967 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: osdmap e1: 0 total, 0 up, 0 in 2024-04-02T04:49:04.967 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:04 smithi152 ceph-mon[29073]: mgrmap e1: no daemons active 2024-04-02T04:49:05.716 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.062+0000 7f636d16c700 1 Processor -- start 2024-04-02T04:49:05.717 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.062+0000 7f636d16c700 1 -- start start 2024-04-02T04:49:05.717 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.062+0000 7f636d16c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6368077b40 0x7f6368076140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:05.717 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.062+0000 7f636d16c700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6368077f50 con 0x7f6368077b40 2024-04-02T04:49:05.717 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.063+0000 7f6366d9d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6368077b40 0x7f6368076140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:05.717 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.063+0000 7f6366d9d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6368077b40 0x7f6368076140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:43302/0 (socket says 172.21.15.152:43302) 2024-04-02T04:49:05.718 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.063+0000 7f6366d9d700 1 -- 172.21.15.152:0/771230316 learned_addr learned my addr 172.21.15.152:0/771230316 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:05.718 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.063+0000 7f6366d9d700 1 -- 172.21.15.152:0/771230316 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6368076680 con 0x7f6368077b40 2024-04-02T04:49:05.718 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.063+0000 7f6366d9d700 1 --2- 172.21.15.152:0/771230316 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6368077b40 0x7f6368076140 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f635c0098e0 tx=0x7f635c009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=a50db36238c3b6d2 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:05.718 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.063+0000 7f6365d9b700 1 -- 172.21.15.152:0/771230316 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f635c00b860 con 0x7f6368077b40 2024-04-02T04:49:05.718 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.063+0000 7f6365d9b700 1 -- 172.21.15.152:0/771230316 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7f635c00be50 con 0x7f6368077b40 2024-04-02T04:49:05.718 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.063+0000 7f6365d9b700 1 -- 172.21.15.152:0/771230316 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f635c010380 con 0x7f6368077b40 2024-04-02T04:49:05.719 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.064+0000 7f636d16c700 1 -- 172.21.15.152:0/771230316 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6368077b40 msgr2=0x7f6368076140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:05.719 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.064+0000 7f636d16c700 1 --2- 172.21.15.152:0/771230316 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6368077b40 0x7f6368076140 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f635c0098e0 tx=0x7f635c009250 comp rx=0 tx=0).stop 2024-04-02T04:49:05.719 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.064+0000 7f636d16c700 1 -- 172.21.15.152:0/771230316 shutdown_connections 2024-04-02T04:49:05.719 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.064+0000 7f636d16c700 1 --2- 172.21.15.152:0/771230316 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6368077b40 0x7f6368076140 unknown :-1 s=CLOSED pgs=1 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:05.719 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.064+0000 7f636d16c700 1 -- 172.21.15.152:0/771230316 >> 172.21.15.152:0/771230316 conn(0x7f63680fe090 msgr2=0x7f63681004c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:05.719 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.064+0000 7f636d16c700 1 -- 172.21.15.152:0/771230316 shutdown_connections 2024-04-02T04:49:05.720 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.064+0000 7f636d16c700 1 -- 172.21.15.152:0/771230316 wait complete. 2024-04-02T04:49:05.720 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.065+0000 7f636d16c700 1 Processor -- start 2024-04-02T04:49:05.720 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.065+0000 7f636d16c700 1 -- start start 2024-04-02T04:49:05.720 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.065+0000 7f636d16c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6368077b40 0x7f63681139c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:05.720 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.065+0000 7f636d16c700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6368113f00 con 0x7f6368077b40 2024-04-02T04:49:05.720 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.065+0000 7f6366d9d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6368077b40 0x7f63681139c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:05.720 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.065+0000 7f6366d9d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6368077b40 0x7f63681139c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:43318/0 (socket says 172.21.15.152:43318) 2024-04-02T04:49:05.721 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.065+0000 7f6366d9d700 1 -- 172.21.15.152:0/3499819996 learned_addr learned my addr 172.21.15.152:0/3499819996 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:05.721 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.065+0000 7f6366d9d700 1 -- 172.21.15.152:0/3499819996 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f635c0094f0 con 0x7f6368077b40 2024-04-02T04:49:05.721 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.065+0000 7f6366d9d700 1 --2- 172.21.15.152:0/3499819996 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6368077b40 0x7f63681139c0 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7f635c006fd0 tx=0x7f635c009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:05.721 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.066+0000 7f634f7fe700 1 -- 172.21.15.152:0/3499819996 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f635c010760 con 0x7f6368077b40 2024-04-02T04:49:05.721 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.066+0000 7f636d16c700 1 -- 172.21.15.152:0/3499819996 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6368114100 con 0x7f6368077b40 2024-04-02T04:49:05.721 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.066+0000 7f634f7fe700 1 -- 172.21.15.152:0/3499819996 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7f635c010d50 con 0x7f6368077b40 2024-04-02T04:49:05.721 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.066+0000 7f634f7fe700 1 -- 172.21.15.152:0/3499819996 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f635c01adc0 con 0x7f6368077b40 2024-04-02T04:49:05.721 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.066+0000 7f636d16c700 1 -- 172.21.15.152:0/3499819996 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6368114c70 con 0x7f6368077b40 2024-04-02T04:49:05.722 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.066+0000 7f634f7fe700 1 -- 172.21.15.152:0/3499819996 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 1) v1 ==== 521+0+0 (secure 0 0 0) 0x7f635c024670 con 0x7f6368077b40 2024-04-02T04:49:05.722 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.066+0000 7f634f7fe700 1 -- 172.21.15.152:0/3499819996 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 721+0+0 (secure 0 0 0) 0x7f635c019840 con 0x7f6368077b40 2024-04-02T04:49:05.722 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.067+0000 7f636d16c700 1 -- 172.21.15.152:0/3499819996 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f636805f5f0 con 0x7f6368077b40 2024-04-02T04:49:05.722 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.069+0000 7f634f7fe700 1 -- 172.21.15.152:0/3499819996 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+71349 (secure 0 0 0) 0x7f635c029060 con 0x7f6368077b40 2024-04-02T04:49:05.722 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.134+0000 7f636d16c700 1 -- 172.21.15.152:0/3499819996 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command([{prefix=config set, name=public_network}] v 0) v1 -- 0x7f6368108b30 con 0x7f6368077b40 2024-04-02T04:49:05.722 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.139+0000 7f634f7fe700 1 -- 172.21.15.152:0/3499819996 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{prefix=config set, name=public_network}]=0 v3)=0 v3) v1 ==== 124+0+0 (secure 0 0 0) 0x7f635c01a5f0 con 0x7f6368077b40 2024-04-02T04:49:05.723 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.140+0000 7f636d16c700 1 -- 172.21.15.152:0/3499819996 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6368077b40 msgr2=0x7f63681139c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:05.723 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.140+0000 7f636d16c700 1 --2- 172.21.15.152:0/3499819996 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6368077b40 0x7f63681139c0 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 crypto rx=0x7f635c006fd0 tx=0x7f635c009250 comp rx=0 tx=0).stop 2024-04-02T04:49:05.723 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.140+0000 7f636d16c700 1 -- 172.21.15.152:0/3499819996 shutdown_connections 2024-04-02T04:49:05.723 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.140+0000 7f636d16c700 1 --2- 172.21.15.152:0/3499819996 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6368077b40 0x7f63681139c0 unknown :-1 s=CLOSED pgs=2 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:05.723 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.140+0000 7f636d16c700 1 -- 172.21.15.152:0/3499819996 >> 172.21.15.152:0/3499819996 conn(0x7f63680fe090 msgr2=0x7f63680fed20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:05.723 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.140+0000 7f636d16c700 1 -- 172.21.15.152:0/3499819996 shutdown_connections 2024-04-02T04:49:05.723 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:05.140+0000 7f636d16c700 1 -- 172.21.15.152:0/3499819996 wait complete. 2024-04-02T04:49:05.723 INFO:teuthology.orchestra.run.smithi152.stdout:Wrote config to /etc/ceph/ceph.conf 2024-04-02T04:49:05.724 INFO:teuthology.orchestra.run.smithi152.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-02T04:49:05.724 INFO:teuthology.orchestra.run.smithi152.stdout:Creating mgr... 2024-04-02T04:49:05.724 INFO:teuthology.orchestra.run.smithi152.stdout:Verifying port 9283 ... 2024-04-02T04:49:05.884 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4@mgr.smithi152.jivdcl 2024-04-02T04:49:05.885 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stderr Failed to reset failed state of unit ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4@mgr.smithi152.jivdcl.service: Unit ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4@mgr.smithi152.jivdcl.service not loaded. 2024-04-02T04:49:06.160 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4.target.wants/ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4@mgr.smithi152.jivdcl.service → /etc/systemd/system/ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4@.service. 2024-04-02T04:49:06.445 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:06 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/3499819996' entity='client.admin' 2024-04-02T04:49:10.327 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-02T04:49:10.327 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout disabled 2024-04-02T04:49:10.334 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-02T04:49:10.334 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout inactive 2024-04-02T04:49:10.334 INFO:teuthology.orchestra.run.smithi152.stdout:firewalld.service is not enabled 2024-04-02T04:49:10.335 INFO:teuthology.orchestra.run.smithi152.stdout:Not possible to enable service . firewalld.service is not available 2024-04-02T04:49:10.343 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-02T04:49:10.343 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout disabled 2024-04-02T04:49:10.350 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-02T04:49:10.350 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout inactive 2024-04-02T04:49:10.351 INFO:teuthology.orchestra.run.smithi152.stdout:firewalld.service is not enabled 2024-04-02T04:49:10.351 INFO:teuthology.orchestra.run.smithi152.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-02T04:49:10.351 INFO:teuthology.orchestra.run.smithi152.stdout:Waiting for mgr to start... 2024-04-02T04:49:10.351 INFO:teuthology.orchestra.run.smithi152.stdout:Waiting for mgr... 2024-04-02T04:49:11.424 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:11 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/2689669696' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-02T04:49:11.767 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 2024-04-02T04:49:11.767 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout { 2024-04-02T04:49:11.767 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "fsid": "2e0055d0-f0ac-11ee-b647-cb9ed24678a4", 2024-04-02T04:49:11.767 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "health": { 2024-04-02T04:49:11.767 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-02T04:49:11.767 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-02T04:49:11.767 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-02T04:49:11.767 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-02T04:49:11.767 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-02T04:49:11.768 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-02T04:49:11.768 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 0 2024-04-02T04:49:11.768 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout ], 2024-04-02T04:49:11.768 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-02T04:49:11.768 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "smithi152" 2024-04-02T04:49:11.768 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout ], 2024-04-02T04:49:11.768 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "quorum_age": 6, 2024-04-02T04:49:11.768 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-02T04:49:11.768 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-02T04:49:11.768 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-02T04:49:11.768 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-02T04:49:11.768 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-02T04:49:11.768 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-02T04:49:11.768 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-02T04:49:11.769 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-02T04:49:11.769 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-02T04:49:11.769 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-02T04:49:11.776 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-02T04:49:11.776 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-02T04:49:11.776 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-02T04:49:11.776 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-02T04:49:11.776 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-02T04:49:11.776 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-02T04:49:11.776 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-02T04:49:11.777 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-02T04:49:11.777 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-02T04:49:11.777 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-02T04:49:11.777 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-02T04:49:11.777 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-02T04:49:11.777 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-02T04:49:11.777 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-02T04:49:11.777 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-02T04:49:11.777 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-02T04:49:11.777 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-02T04:49:11.777 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-02T04:49:11.777 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-02T04:49:11.777 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-02T04:49:11.777 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-02T04:49:11.778 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-02T04:49:11.778 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-02T04:49:11.778 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-02T04:49:11.778 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-02T04:49:11.778 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "restful" 2024-04-02T04:49:11.778 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout ], 2024-04-02T04:49:11.778 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-02T04:49:11.778 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-02T04:49:11.778 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-02T04:49:11.778 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-02T04:49:11.778 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "modified": "2024-04-02T04:48:59.746632+0000", 2024-04-02T04:49:11.778 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-02T04:49:11.778 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-02T04:49:11.778 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-02T04:49:11.779 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout } 2024-04-02T04:49:11.779 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.118+0000 7f639bed0700 1 Processor -- start 2024-04-02T04:49:11.779 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.118+0000 7f639bed0700 1 -- start start 2024-04-02T04:49:11.779 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.118+0000 7f639bed0700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f63941028c0 0x7f6394102cd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:11.779 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.118+0000 7f639bed0700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6394103210 con 0x7f63941028c0 2024-04-02T04:49:11.779 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.119+0000 7f6399c6c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f63941028c0 0x7f6394102cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:11.779 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.119+0000 7f6399c6c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f63941028c0 0x7f6394102cd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:53328/0 (socket says 172.21.15.152:53328) 2024-04-02T04:49:11.779 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.119+0000 7f6399c6c700 1 -- 172.21.15.152:0/750594456 learned_addr learned my addr 172.21.15.152:0/750594456 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:11.779 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.119+0000 7f6399c6c700 1 -- 172.21.15.152:0/750594456 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6394103350 con 0x7f63941028c0 2024-04-02T04:49:11.779 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.119+0000 7f6399c6c700 1 --2- 172.21.15.152:0/750594456 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f63941028c0 0x7f6394102cd0 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7f63840098e0 tx=0x7f6384009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9c71ca9bf394d11c server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:11.779 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.119+0000 7f6398c6a700 1 -- 172.21.15.152:0/750594456 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f638400b860 con 0x7f63941028c0 2024-04-02T04:49:11.780 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.119+0000 7f6398c6a700 1 -- 172.21.15.152:0/750594456 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7f638400be50 con 0x7f63941028c0 2024-04-02T04:49:11.780 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.119+0000 7f6398c6a700 1 -- 172.21.15.152:0/750594456 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6384010380 con 0x7f63941028c0 2024-04-02T04:49:11.780 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.120+0000 7f639bed0700 1 -- 172.21.15.152:0/750594456 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f63941028c0 msgr2=0x7f6394102cd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:11.780 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.120+0000 7f639bed0700 1 --2- 172.21.15.152:0/750594456 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f63941028c0 0x7f6394102cd0 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7f63840098e0 tx=0x7f6384009250 comp rx=0 tx=0).stop 2024-04-02T04:49:11.780 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.120+0000 7f639bed0700 1 -- 172.21.15.152:0/750594456 shutdown_connections 2024-04-02T04:49:11.780 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.120+0000 7f639bed0700 1 --2- 172.21.15.152:0/750594456 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f63941028c0 0x7f6394102cd0 unknown :-1 s=CLOSED pgs=5 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:11.780 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.120+0000 7f639bed0700 1 -- 172.21.15.152:0/750594456 >> 172.21.15.152:0/750594456 conn(0x7f63940fe080 msgr2=0x7f6394100490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:11.780 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.120+0000 7f639bed0700 1 -- 172.21.15.152:0/750594456 shutdown_connections 2024-04-02T04:49:11.780 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.120+0000 7f639bed0700 1 -- 172.21.15.152:0/750594456 wait complete. 2024-04-02T04:49:11.780 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.121+0000 7f639bed0700 1 Processor -- start 2024-04-02T04:49:11.780 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.121+0000 7f639bed0700 1 -- start start 2024-04-02T04:49:11.780 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.121+0000 7f639bed0700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f63941028c0 0x7f6394078c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:11.780 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.121+0000 7f639bed0700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6394079180 con 0x7f63941028c0 2024-04-02T04:49:11.780 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.121+0000 7f6399c6c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f63941028c0 0x7f6394078c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:11.780 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.121+0000 7f6399c6c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f63941028c0 0x7f6394078c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:53336/0 (socket says 172.21.15.152:53336) 2024-04-02T04:49:11.780 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.121+0000 7f6399c6c700 1 -- 172.21.15.152:0/2689669696 learned_addr learned my addr 172.21.15.152:0/2689669696 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:11.781 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.121+0000 7f6399c6c700 1 -- 172.21.15.152:0/2689669696 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f63840094f0 con 0x7f63941028c0 2024-04-02T04:49:11.781 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.122+0000 7f6399c6c700 1 --2- 172.21.15.152:0/2689669696 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f63941028c0 0x7f6394078c40 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7f638400b440 tx=0x7f6384009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:11.781 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.122+0000 7f638a7fc700 1 -- 172.21.15.152:0/2689669696 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6384010760 con 0x7f63941028c0 2024-04-02T04:49:11.781 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.122+0000 7f638a7fc700 1 -- 172.21.15.152:0/2689669696 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7f6384010d50 con 0x7f63941028c0 2024-04-02T04:49:11.781 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.122+0000 7f639bed0700 1 -- 172.21.15.152:0/2689669696 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f63940771f0 con 0x7f63941028c0 2024-04-02T04:49:11.781 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.122+0000 7f638a7fc700 1 -- 172.21.15.152:0/2689669696 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f638401ae20 con 0x7f63941028c0 2024-04-02T04:49:11.781 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.122+0000 7f639bed0700 1 -- 172.21.15.152:0/2689669696 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f63940776e0 con 0x7f63941028c0 2024-04-02T04:49:11.781 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.122+0000 7f638a7fc700 1 -- 172.21.15.152:0/2689669696 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 1) v1 ==== 521+0+0 (secure 0 0 0) 0x7f638402e6c0 con 0x7f63941028c0 2024-04-02T04:49:11.781 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.122+0000 7f638a7fc700 1 -- 172.21.15.152:0/2689669696 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 721+0+0 (secure 0 0 0) 0x7f6384024950 con 0x7f63941028c0 2024-04-02T04:49:11.781 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.123+0000 7f639bed0700 1 -- 172.21.15.152:0/2689669696 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f639404bed0 con 0x7f63941028c0 2024-04-02T04:49:11.781 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.125+0000 7f638a7fc700 1 -- 172.21.15.152:0/2689669696 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+71349 (secure 0 0 0) 0x7f6384029070 con 0x7f63941028c0 2024-04-02T04:49:11.781 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.196+0000 7f639bed0700 1 -- 172.21.15.152:0/2689669696 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) v1 -- 0x7f6394077920 con 0x7f63941028c0 2024-04-02T04:49:11.781 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.197+0000 7f638a7fc700 1 -- 172.21.15.152:0/2689669696 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) v1 ==== 79+0+1248 (secure 0 0 0) 0x7f638401a6a0 con 0x7f63941028c0 2024-04-02T04:49:11.781 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.197+0000 7f639bed0700 1 -- 172.21.15.152:0/2689669696 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f63941028c0 msgr2=0x7f6394078c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:11.782 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.197+0000 7f639bed0700 1 --2- 172.21.15.152:0/2689669696 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f63941028c0 0x7f6394078c40 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7f638400b440 tx=0x7f6384009250 comp rx=0 tx=0).stop 2024-04-02T04:49:11.782 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.198+0000 7f639bed0700 1 -- 172.21.15.152:0/2689669696 shutdown_connections 2024-04-02T04:49:11.782 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.198+0000 7f639bed0700 1 --2- 172.21.15.152:0/2689669696 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f63941028c0 0x7f6394078c40 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:11.782 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.198+0000 7f639bed0700 1 -- 172.21.15.152:0/2689669696 >> 172.21.15.152:0/2689669696 conn(0x7f63940fe080 msgr2=0x7f63940febd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:11.782 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.198+0000 7f639bed0700 1 -- 172.21.15.152:0/2689669696 shutdown_connections 2024-04-02T04:49:11.782 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:11.198+0000 7f639bed0700 1 -- 172.21.15.152:0/2689669696 wait complete. 2024-04-02T04:49:11.782 INFO:teuthology.orchestra.run.smithi152.stdout:mgr not available, waiting (1/15)... 2024-04-02T04:49:14.659 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:14 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/3211511772' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-02T04:49:14.872 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 2024-04-02T04:49:14.872 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout { 2024-04-02T04:49:14.872 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "fsid": "2e0055d0-f0ac-11ee-b647-cb9ed24678a4", 2024-04-02T04:49:14.872 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "health": { 2024-04-02T04:49:14.872 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-02T04:49:14.872 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-02T04:49:14.873 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-02T04:49:14.873 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-02T04:49:14.873 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-02T04:49:14.873 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-02T04:49:14.873 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 0 2024-04-02T04:49:14.873 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout ], 2024-04-02T04:49:14.873 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-02T04:49:14.873 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "smithi152" 2024-04-02T04:49:14.873 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout ], 2024-04-02T04:49:14.873 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "quorum_age": 9, 2024-04-02T04:49:14.873 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-02T04:49:14.873 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-02T04:49:14.873 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-02T04:49:14.873 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-02T04:49:14.874 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-02T04:49:14.874 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-02T04:49:14.874 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-02T04:49:14.874 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-02T04:49:14.874 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-02T04:49:14.874 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-02T04:49:14.874 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-02T04:49:14.874 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-02T04:49:14.874 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-02T04:49:14.874 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-02T04:49:14.874 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-02T04:49:14.874 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-02T04:49:14.874 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-02T04:49:14.874 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-02T04:49:14.875 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-02T04:49:14.875 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-02T04:49:14.875 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-02T04:49:14.875 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-02T04:49:14.875 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-02T04:49:14.875 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-02T04:49:14.875 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-02T04:49:14.875 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-02T04:49:14.875 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-02T04:49:14.875 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-02T04:49:14.875 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-02T04:49:14.875 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-02T04:49:14.875 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-02T04:49:14.875 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-02T04:49:14.875 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-02T04:49:14.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-02T04:49:14.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-02T04:49:14.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "restful" 2024-04-02T04:49:14.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout ], 2024-04-02T04:49:14.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-02T04:49:14.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-02T04:49:14.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-02T04:49:14.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-02T04:49:14.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "modified": "2024-04-02T04:48:59.746632+0000", 2024-04-02T04:49:14.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-02T04:49:14.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-02T04:49:14.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-02T04:49:14.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout } 2024-04-02T04:49:14.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.202+0000 7f84a3355700 1 Processor -- start 2024-04-02T04:49:14.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.202+0000 7f84a3355700 1 -- start start 2024-04-02T04:49:14.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.202+0000 7f84a3355700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f849c102890 0x7f849c102ca0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:14.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.202+0000 7f84a3355700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f849c1031e0 con 0x7f849c102890 2024-04-02T04:49:14.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.202+0000 7f84a10f1700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f849c102890 0x7f849c102ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:14.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.202+0000 7f84a10f1700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f849c102890 0x7f849c102ca0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:53346/0 (socket says 172.21.15.152:53346) 2024-04-02T04:49:14.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.202+0000 7f84a10f1700 1 -- 172.21.15.152:0/3666931376 learned_addr learned my addr 172.21.15.152:0/3666931376 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:14.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.203+0000 7f84a10f1700 1 -- 172.21.15.152:0/3666931376 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f849c103320 con 0x7f849c102890 2024-04-02T04:49:14.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.203+0000 7f84a10f1700 1 --2- 172.21.15.152:0/3666931376 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f849c102890 0x7f849c102ca0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f848c0098e0 tx=0x7f848c009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f4b0eee2c56703bf server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:14.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.203+0000 7f8493fff700 1 -- 172.21.15.152:0/3666931376 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f848c00b860 con 0x7f849c102890 2024-04-02T04:49:14.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.203+0000 7f8493fff700 1 -- 172.21.15.152:0/3666931376 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7f848c00be50 con 0x7f849c102890 2024-04-02T04:49:14.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.203+0000 7f8493fff700 1 -- 172.21.15.152:0/3666931376 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f848c010380 con 0x7f849c102890 2024-04-02T04:49:14.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.204+0000 7f84a3355700 1 -- 172.21.15.152:0/3666931376 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f849c102890 msgr2=0x7f849c102ca0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:14.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.204+0000 7f84a3355700 1 --2- 172.21.15.152:0/3666931376 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f849c102890 0x7f849c102ca0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f848c0098e0 tx=0x7f848c009250 comp rx=0 tx=0).stop 2024-04-02T04:49:14.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.204+0000 7f84a3355700 1 -- 172.21.15.152:0/3666931376 shutdown_connections 2024-04-02T04:49:14.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.204+0000 7f84a3355700 1 --2- 172.21.15.152:0/3666931376 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f849c102890 0x7f849c102ca0 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:14.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.204+0000 7f84a3355700 1 -- 172.21.15.152:0/3666931376 >> 172.21.15.152:0/3666931376 conn(0x7f849c0fe0a0 msgr2=0x7f849c1004b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:14.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.204+0000 7f84a3355700 1 -- 172.21.15.152:0/3666931376 shutdown_connections 2024-04-02T04:49:14.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.204+0000 7f84a3355700 1 -- 172.21.15.152:0/3666931376 wait complete. 2024-04-02T04:49:14.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.204+0000 7f84a3355700 1 Processor -- start 2024-04-02T04:49:14.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.204+0000 7f84a3355700 1 -- start start 2024-04-02T04:49:14.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.204+0000 7f84a3355700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f849c102890 0x7f849c1116e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:14.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.204+0000 7f84a3355700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f849c111c20 con 0x7f849c102890 2024-04-02T04:49:14.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.205+0000 7f84a10f1700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f849c102890 0x7f849c1116e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:14.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.205+0000 7f84a10f1700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f849c102890 0x7f849c1116e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:53352/0 (socket says 172.21.15.152:53352) 2024-04-02T04:49:14.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.205+0000 7f84a10f1700 1 -- 172.21.15.152:0/3211511772 learned_addr learned my addr 172.21.15.152:0/3211511772 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:14.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.205+0000 7f84a10f1700 1 -- 172.21.15.152:0/3211511772 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f848c0094f0 con 0x7f849c102890 2024-04-02T04:49:14.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.205+0000 7f84a10f1700 1 --2- 172.21.15.152:0/3211511772 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f849c102890 0x7f849c1116e0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f848c00b440 tx=0x7f848c009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:14.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.205+0000 7f8491ffb700 1 -- 172.21.15.152:0/3211511772 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f848c010760 con 0x7f849c102890 2024-04-02T04:49:14.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.205+0000 7f8491ffb700 1 -- 172.21.15.152:0/3211511772 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7f848c010d50 con 0x7f849c102890 2024-04-02T04:49:14.879 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.205+0000 7f84a3355700 1 -- 172.21.15.152:0/3211511772 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f849c111e20 con 0x7f849c102890 2024-04-02T04:49:14.879 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.205+0000 7f8491ffb700 1 -- 172.21.15.152:0/3211511772 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f848c01ae20 con 0x7f849c102890 2024-04-02T04:49:14.879 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.205+0000 7f84a3355700 1 -- 172.21.15.152:0/3211511772 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f849c112990 con 0x7f849c102890 2024-04-02T04:49:14.879 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.206+0000 7f8491ffb700 1 -- 172.21.15.152:0/3211511772 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 1) v1 ==== 521+0+0 (secure 0 0 0) 0x7f848c02e6c0 con 0x7f849c102890 2024-04-02T04:49:14.879 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.206+0000 7f8491ffb700 1 -- 172.21.15.152:0/3211511772 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 721+0+0 (secure 0 0 0) 0x7f848c0248c0 con 0x7f849c102890 2024-04-02T04:49:14.879 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.206+0000 7f84a3355700 1 -- 172.21.15.152:0/3211511772 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f849c05f5f0 con 0x7f849c102890 2024-04-02T04:49:14.879 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.210+0000 7f8491ffb700 1 -- 172.21.15.152:0/3211511772 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+71349 (secure 0 0 0) 0x7f848c029070 con 0x7f849c102890 2024-04-02T04:49:14.879 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.307+0000 7f84a3355700 1 -- 172.21.15.152:0/3211511772 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) v1 -- 0x7f849c104610 con 0x7f849c102890 2024-04-02T04:49:14.879 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.308+0000 7f8491ffb700 1 -- 172.21.15.152:0/3211511772 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) v1 ==== 79+0+1248 (secure 0 0 0) 0x7f848c01a6a0 con 0x7f849c102890 2024-04-02T04:49:14.879 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.309+0000 7f84a3355700 1 -- 172.21.15.152:0/3211511772 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f849c102890 msgr2=0x7f849c1116e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:14.879 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.309+0000 7f84a3355700 1 --2- 172.21.15.152:0/3211511772 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f849c102890 0x7f849c1116e0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f848c00b440 tx=0x7f848c009250 comp rx=0 tx=0).stop 2024-04-02T04:49:14.879 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.309+0000 7f84a3355700 1 -- 172.21.15.152:0/3211511772 shutdown_connections 2024-04-02T04:49:14.879 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.309+0000 7f84a3355700 1 --2- 172.21.15.152:0/3211511772 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f849c102890 0x7f849c1116e0 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:14.879 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.309+0000 7f84a3355700 1 -- 172.21.15.152:0/3211511772 >> 172.21.15.152:0/3211511772 conn(0x7f849c0fe0a0 msgr2=0x7f849c0febf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:14.879 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.309+0000 7f84a3355700 1 -- 172.21.15.152:0/3211511772 shutdown_connections 2024-04-02T04:49:14.880 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:14.309+0000 7f84a3355700 1 -- 172.21.15.152:0/3211511772 wait complete. 2024-04-02T04:49:14.880 INFO:teuthology.orchestra.run.smithi152.stdout:mgr not available, waiting (2/15)... 2024-04-02T04:49:16.006 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:15 smithi152 ceph-mon[29073]: Activating manager daemon smithi152.jivdcl 2024-04-02T04:49:16.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:15 smithi152 ceph-mon[29073]: mgrmap e2: smithi152.jivdcl(active, starting, since 0.00306421s) 2024-04-02T04:49:16.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:15 smithi152 ceph-mon[29073]: from='mgr.14100 172.21.15.152:0/2884285300' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T04:49:16.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:15 smithi152 ceph-mon[29073]: from='mgr.14100 172.21.15.152:0/2884285300' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T04:49:16.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:15 smithi152 ceph-mon[29073]: from='mgr.14100 172.21.15.152:0/2884285300' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T04:49:16.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:15 smithi152 ceph-mon[29073]: from='mgr.14100 172.21.15.152:0/2884285300' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi152"}]: dispatch 2024-04-02T04:49:16.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:15 smithi152 ceph-mon[29073]: from='mgr.14100 172.21.15.152:0/2884285300' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mgr metadata", "who": "smithi152.jivdcl", "id": "smithi152.jivdcl"}]: dispatch 2024-04-02T04:49:16.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:15 smithi152 ceph-mon[29073]: Manager daemon smithi152.jivdcl is now available 2024-04-02T04:49:16.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:15 smithi152 ceph-mon[29073]: from='mgr.14100 172.21.15.152:0/2884285300' entity='mgr.smithi152.jivdcl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi152.jivdcl/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:49:16.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:15 smithi152 ceph-mon[29073]: from='mgr.14100 172.21.15.152:0/2884285300' entity='mgr.smithi152.jivdcl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi152.jivdcl/trash_purge_schedule"}]: dispatch 2024-04-02T04:49:16.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:15 smithi152 ceph-mon[29073]: from='mgr.14100 172.21.15.152:0/2884285300' entity='mgr.smithi152.jivdcl' 2024-04-02T04:49:16.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:15 smithi152 ceph-mon[29073]: from='mgr.14100 172.21.15.152:0/2884285300' entity='mgr.smithi152.jivdcl' 2024-04-02T04:49:16.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:15 smithi152 ceph-mon[29073]: from='mgr.14100 172.21.15.152:0/2884285300' entity='mgr.smithi152.jivdcl' 2024-04-02T04:49:18.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:17 smithi152 ceph-mon[29073]: mgrmap e3: smithi152.jivdcl(active, since 1.00706s) 2024-04-02T04:49:19.230 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:18 smithi152 ceph-mon[29073]: mgrmap e4: smithi152.jivdcl(active, since 2s) 2024-04-02T04:49:20.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:20 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/2756983035' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-02T04:49:22.505 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 2024-04-02T04:49:22.505 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout { 2024-04-02T04:49:22.505 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "fsid": "2e0055d0-f0ac-11ee-b647-cb9ed24678a4", 2024-04-02T04:49:22.505 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "health": { 2024-04-02T04:49:22.505 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-02T04:49:22.505 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-02T04:49:22.505 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-02T04:49:22.505 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-02T04:49:22.505 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-02T04:49:22.505 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-02T04:49:22.506 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 0 2024-04-02T04:49:22.506 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout ], 2024-04-02T04:49:22.506 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-02T04:49:22.506 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "smithi152" 2024-04-02T04:49:22.506 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout ], 2024-04-02T04:49:22.506 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "quorum_age": 15, 2024-04-02T04:49:22.506 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-02T04:49:22.506 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-02T04:49:22.506 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-04-02T04:49:22.506 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-02T04:49:22.506 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-02T04:49:22.506 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-02T04:49:22.506 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-02T04:49:22.506 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-02T04:49:22.507 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-02T04:49:22.507 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-02T04:49:22.518 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-02T04:49:22.518 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-02T04:49:22.518 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-02T04:49:22.518 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-02T04:49:22.518 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-02T04:49:22.518 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-02T04:49:22.518 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-02T04:49:22.518 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-02T04:49:22.518 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-02T04:49:22.518 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-02T04:49:22.519 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-02T04:49:22.519 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-02T04:49:22.519 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-02T04:49:22.519 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-02T04:49:22.519 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-02T04:49:22.519 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-02T04:49:22.519 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-02T04:49:22.519 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-02T04:49:22.519 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-02T04:49:22.519 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-02T04:49:22.519 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-02T04:49:22.519 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-02T04:49:22.519 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-02T04:49:22.520 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-02T04:49:22.520 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-02T04:49:22.520 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "restful" 2024-04-02T04:49:22.520 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout ], 2024-04-02T04:49:22.520 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-02T04:49:22.520 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-02T04:49:22.520 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-02T04:49:22.520 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-02T04:49:22.520 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "modified": "2024-04-02T04:48:59.746632+0000", 2024-04-02T04:49:22.520 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-02T04:49:22.520 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout }, 2024-04-02T04:49:22.520 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-02T04:49:22.520 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout } 2024-04-02T04:49:22.520 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.216+0000 7eff3a672700 1 Processor -- start 2024-04-02T04:49:22.521 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.216+0000 7eff3a672700 1 -- start start 2024-04-02T04:49:22.521 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.216+0000 7eff3a672700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7eff34071250 0x7eff34105540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:22.521 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.216+0000 7eff3a672700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7eff34071660 con 0x7eff34071250 2024-04-02T04:49:22.521 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.216+0000 7eff33fff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7eff34071250 0x7eff34105540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:22.521 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.216+0000 7eff33fff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7eff34071250 0x7eff34105540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:53422/0 (socket says 172.21.15.152:53422) 2024-04-02T04:49:22.521 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.216+0000 7eff33fff700 1 -- 172.21.15.152:0/981574902 learned_addr learned my addr 172.21.15.152:0/981574902 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:22.521 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.217+0000 7eff33fff700 1 -- 172.21.15.152:0/981574902 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7eff34105a80 con 0x7eff34071250 2024-04-02T04:49:22.521 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.217+0000 7eff33fff700 1 --2- 172.21.15.152:0/981574902 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7eff34071250 0x7eff34105540 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7eff240098e0 tx=0x7eff24009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=be60f32da479a5c5 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:22.521 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.217+0000 7eff32ffd700 1 -- 172.21.15.152:0/981574902 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7eff2400b860 con 0x7eff34071250 2024-04-02T04:49:22.521 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.217+0000 7eff32ffd700 1 -- 172.21.15.152:0/981574902 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7eff2400be50 con 0x7eff34071250 2024-04-02T04:49:22.521 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.217+0000 7eff32ffd700 1 -- 172.21.15.152:0/981574902 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7eff24010380 con 0x7eff34071250 2024-04-02T04:49:22.521 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.217+0000 7eff3a672700 1 -- 172.21.15.152:0/981574902 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7eff34071250 msgr2=0x7eff34105540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:22.521 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.217+0000 7eff3a672700 1 --2- 172.21.15.152:0/981574902 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7eff34071250 0x7eff34105540 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7eff240098e0 tx=0x7eff24009250 comp rx=0 tx=0).stop 2024-04-02T04:49:22.522 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.217+0000 7eff3a672700 1 -- 172.21.15.152:0/981574902 shutdown_connections 2024-04-02T04:49:22.522 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.217+0000 7eff3a672700 1 --2- 172.21.15.152:0/981574902 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7eff34071250 0x7eff34105540 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:22.522 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.217+0000 7eff3a672700 1 -- 172.21.15.152:0/981574902 >> 172.21.15.152:0/981574902 conn(0x7eff340fcfb0 msgr2=0x7eff340ff3c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:22.522 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.218+0000 7eff3a672700 1 -- 172.21.15.152:0/981574902 shutdown_connections 2024-04-02T04:49:22.522 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.218+0000 7eff3a672700 1 -- 172.21.15.152:0/981574902 wait complete. 2024-04-02T04:49:22.522 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.218+0000 7eff3a672700 1 Processor -- start 2024-04-02T04:49:22.522 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.218+0000 7eff3a672700 1 -- start start 2024-04-02T04:49:22.522 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.219+0000 7eff3a672700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7eff34071250 0x7eff34111760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:22.522 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.219+0000 7eff3a672700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7eff34111ca0 con 0x7eff34071250 2024-04-02T04:49:22.522 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.219+0000 7eff33fff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7eff34071250 0x7eff34111760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:22.522 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.219+0000 7eff33fff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7eff34071250 0x7eff34111760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:53426/0 (socket says 172.21.15.152:53426) 2024-04-02T04:49:22.522 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.219+0000 7eff33fff700 1 -- 172.21.15.152:0/2756983035 learned_addr learned my addr 172.21.15.152:0/2756983035 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:22.522 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.219+0000 7eff33fff700 1 -- 172.21.15.152:0/2756983035 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7eff240094f0 con 0x7eff34071250 2024-04-02T04:49:22.522 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.219+0000 7eff33fff700 1 --2- 172.21.15.152:0/2756983035 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7eff34071250 0x7eff34111760 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7eff24011bd0 tx=0x7eff24009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:22.522 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.219+0000 7eff30ff9700 1 -- 172.21.15.152:0/2756983035 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7eff240107c0 con 0x7eff34071250 2024-04-02T04:49:22.523 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.219+0000 7eff30ff9700 1 -- 172.21.15.152:0/2756983035 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7eff24010db0 con 0x7eff34071250 2024-04-02T04:49:22.523 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.219+0000 7eff3a672700 1 -- 172.21.15.152:0/2756983035 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7eff34111ea0 con 0x7eff34071250 2024-04-02T04:49:22.523 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.219+0000 7eff30ff9700 1 -- 172.21.15.152:0/2756983035 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7eff2401ad00 con 0x7eff34071250 2024-04-02T04:49:22.523 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.219+0000 7eff3a672700 1 -- 172.21.15.152:0/2756983035 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7eff34112a10 con 0x7eff34071250 2024-04-02T04:49:22.523 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.220+0000 7eff30ff9700 1 -- 172.21.15.152:0/2756983035 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 4) v1 ==== 43357+0+0 (secure 0 0 0) 0x7eff240193c0 con 0x7eff34071250 2024-04-02T04:49:22.523 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.220+0000 7eff30ff9700 1 --2- 172.21.15.152:0/2756983035 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7eff1c036790 0x7eff1c038c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:22.523 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.220+0000 7eff30ff9700 1 -- 172.21.15.152:0/2756983035 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 721+0+0 (secure 0 0 0) 0x7eff24049fb0 con 0x7eff34071250 2024-04-02T04:49:22.523 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.220+0000 7eff3a672700 1 -- 172.21.15.152:0/2756983035 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7eff3405f5f0 con 0x7eff34071250 2024-04-02T04:49:22.523 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.220+0000 7eff337fe700 1 --2- 172.21.15.152:0/2756983035 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7eff1c036790 0x7eff1c038c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:22.523 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.221+0000 7eff337fe700 1 --2- 172.21.15.152:0/2756983035 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7eff1c036790 0x7eff1c038c40 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7eff28009990 tx=0x7eff28006e40 comp rx=0 tx=0).ready entity=mgr.14100 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:22.523 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.225+0000 7eff30ff9700 1 -- 172.21.15.152:0/2756983035 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7eff24020070 con 0x7eff34071250 2024-04-02T04:49:22.523 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.488+0000 7eff3a672700 1 -- 172.21.15.152:0/2756983035 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) v1 -- 0x7eff3404bed0 con 0x7eff34071250 2024-04-02T04:49:22.523 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.489+0000 7eff30ff9700 1 -- 172.21.15.152:0/2756983035 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) v1 ==== 79+0+1248 (secure 0 0 0) 0x7eff2402c300 con 0x7eff34071250 2024-04-02T04:49:22.523 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.491+0000 7eff3a672700 1 -- 172.21.15.152:0/2756983035 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7eff1c036790 msgr2=0x7eff1c038c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:22.523 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.491+0000 7eff3a672700 1 --2- 172.21.15.152:0/2756983035 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7eff1c036790 0x7eff1c038c40 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7eff28009990 tx=0x7eff28006e40 comp rx=0 tx=0).stop 2024-04-02T04:49:22.524 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.491+0000 7eff3a672700 1 -- 172.21.15.152:0/2756983035 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7eff34071250 msgr2=0x7eff34111760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:22.524 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.491+0000 7eff3a672700 1 --2- 172.21.15.152:0/2756983035 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7eff34071250 0x7eff34111760 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7eff24011bd0 tx=0x7eff24009250 comp rx=0 tx=0).stop 2024-04-02T04:49:22.524 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.491+0000 7eff3a672700 1 -- 172.21.15.152:0/2756983035 shutdown_connections 2024-04-02T04:49:22.524 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.491+0000 7eff3a672700 1 --2- 172.21.15.152:0/2756983035 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7eff1c036790 0x7eff1c038c40 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:22.524 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.491+0000 7eff3a672700 1 --2- 172.21.15.152:0/2756983035 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7eff34071250 0x7eff34111760 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:22.524 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.491+0000 7eff3a672700 1 -- 172.21.15.152:0/2756983035 >> 172.21.15.152:0/2756983035 conn(0x7eff340fcfb0 msgr2=0x7eff340fdbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:22.524 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.491+0000 7eff3a672700 1 -- 172.21.15.152:0/2756983035 shutdown_connections 2024-04-02T04:49:22.524 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:20.491+0000 7eff3a672700 1 -- 172.21.15.152:0/2756983035 wait complete. 2024-04-02T04:49:22.524 INFO:teuthology.orchestra.run.smithi152.stdout:mgr is available 2024-04-02T04:49:23.323 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:23 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/3723917424' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-02T04:49:23.582 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 2024-04-02T04:49:23.582 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout [global] 2024-04-02T04:49:23.582 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout fsid = 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 2024-04-02T04:49:23.582 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-02T04:49:23.582 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-02T04:49:23.582 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-02T04:49:23.582 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-02T04:49:23.583 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 2024-04-02T04:49:23.583 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-02T04:49:23.583 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-02T04:49:23.583 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 2024-04-02T04:49:23.583 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout [osd] 2024-04-02T04:49:23.583 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mon_osd_backfillfull_ratio = 0.85 2024-04-02T04:49:23.583 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mon_osd_full_ratio = 0.9 2024-04-02T04:49:23.583 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout mon_osd_nearfull_ratio = 0.8 2024-04-02T04:49:23.583 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-02T04:49:23.583 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-02T04:49:23.583 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.874+0000 7f6b3b85b700 1 Processor -- start 2024-04-02T04:49:23.583 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.874+0000 7f6b3b85b700 1 -- start start 2024-04-02T04:49:23.583 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.874+0000 7f6b3b85b700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6b34102910 0x7f6b34102d20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:23.583 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.874+0000 7f6b3b85b700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6b34103260 con 0x7f6b34102910 2024-04-02T04:49:23.583 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.875+0000 7f6b395f7700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6b34102910 0x7f6b34102d20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:23.583 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.875+0000 7f6b395f7700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6b34102910 0x7f6b34102d20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:52822/0 (socket says 172.21.15.152:52822) 2024-04-02T04:49:23.584 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.875+0000 7f6b395f7700 1 -- 172.21.15.152:0/242598944 learned_addr learned my addr 172.21.15.152:0/242598944 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:23.584 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.875+0000 7f6b395f7700 1 -- 172.21.15.152:0/242598944 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6b341033a0 con 0x7f6b34102910 2024-04-02T04:49:23.584 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.875+0000 7f6b395f7700 1 --2- 172.21.15.152:0/242598944 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6b34102910 0x7f6b34102d20 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f6b30009e50 tx=0x7f6b30009840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=aed7a12d61d1a030 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:23.584 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.875+0000 7f6b2bfff700 1 -- 172.21.15.152:0/242598944 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6b300092e0 con 0x7f6b34102910 2024-04-02T04:49:23.584 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.875+0000 7f6b2bfff700 1 -- 172.21.15.152:0/242598944 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7f6b30012430 con 0x7f6b34102910 2024-04-02T04:49:23.584 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.875+0000 7f6b2bfff700 1 -- 172.21.15.152:0/242598944 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6b300075c0 con 0x7f6b34102910 2024-04-02T04:49:23.584 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.876+0000 7f6b3b85b700 1 -- 172.21.15.152:0/242598944 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6b34102910 msgr2=0x7f6b34102d20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:23.584 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.876+0000 7f6b3b85b700 1 --2- 172.21.15.152:0/242598944 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6b34102910 0x7f6b34102d20 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7f6b30009e50 tx=0x7f6b30009840 comp rx=0 tx=0).stop 2024-04-02T04:49:23.584 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.876+0000 7f6b3b85b700 1 -- 172.21.15.152:0/242598944 shutdown_connections 2024-04-02T04:49:23.584 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.876+0000 7f6b3b85b700 1 --2- 172.21.15.152:0/242598944 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6b34102910 0x7f6b34102d20 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:23.584 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.876+0000 7f6b3b85b700 1 -- 172.21.15.152:0/242598944 >> 172.21.15.152:0/242598944 conn(0x7f6b340fe090 msgr2=0x7f6b341004a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:23.584 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.876+0000 7f6b3b85b700 1 -- 172.21.15.152:0/242598944 shutdown_connections 2024-04-02T04:49:23.584 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.876+0000 7f6b3b85b700 1 -- 172.21.15.152:0/242598944 wait complete. 2024-04-02T04:49:23.584 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.876+0000 7f6b3b85b700 1 Processor -- start 2024-04-02T04:49:23.584 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.876+0000 7f6b3b85b700 1 -- start start 2024-04-02T04:49:23.584 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.877+0000 7f6b3b85b700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6b34102910 0x7f6b34111750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:23.585 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.877+0000 7f6b3b85b700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6b34111c90 con 0x7f6b34102910 2024-04-02T04:49:23.585 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.877+0000 7f6b395f7700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6b34102910 0x7f6b34111750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:23.585 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.877+0000 7f6b395f7700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6b34102910 0x7f6b34111750 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:52828/0 (socket says 172.21.15.152:52828) 2024-04-02T04:49:23.585 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.877+0000 7f6b395f7700 1 -- 172.21.15.152:0/3723917424 learned_addr learned my addr 172.21.15.152:0/3723917424 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:23.585 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.877+0000 7f6b395f7700 1 -- 172.21.15.152:0/3723917424 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6b30009ae0 con 0x7f6b34102910 2024-04-02T04:49:23.585 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.877+0000 7f6b395f7700 1 --2- 172.21.15.152:0/3723917424 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6b34102910 0x7f6b34111750 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f6b30012da0 tx=0x7f6b30009840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:23.585 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.877+0000 7f6b29ffb700 1 -- 172.21.15.152:0/3723917424 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6b300092e0 con 0x7f6b34102910 2024-04-02T04:49:23.585 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.877+0000 7f6b29ffb700 1 -- 172.21.15.152:0/3723917424 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7f6b30007d10 con 0x7f6b34102910 2024-04-02T04:49:23.585 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.877+0000 7f6b29ffb700 1 -- 172.21.15.152:0/3723917424 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6b3001c8c0 con 0x7f6b34102910 2024-04-02T04:49:23.585 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.877+0000 7f6b3b85b700 1 -- 172.21.15.152:0/3723917424 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6b34111e90 con 0x7f6b34102910 2024-04-02T04:49:23.585 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.877+0000 7f6b3b85b700 1 -- 172.21.15.152:0/3723917424 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6b34112a00 con 0x7f6b34102910 2024-04-02T04:49:23.585 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.878+0000 7f6b29ffb700 1 -- 172.21.15.152:0/3723917424 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 4) v1 ==== 43357+0+0 (secure 0 0 0) 0x7f6b30019070 con 0x7f6b34102910 2024-04-02T04:49:23.585 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.878+0000 7f6b29ffb700 1 --2- 172.21.15.152:0/3723917424 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f6b20036790 0x7f6b20038c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:23.585 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.878+0000 7f6b29ffb700 1 -- 172.21.15.152:0/3723917424 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 721+0+0 (secure 0 0 0) 0x7f6b30049ce0 con 0x7f6b34102910 2024-04-02T04:49:23.585 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.878+0000 7f6b3b85b700 1 -- 172.21.15.152:0/3723917424 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6b3405f5f0 con 0x7f6b34102910 2024-04-02T04:49:23.585 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.878+0000 7f6b38df6700 1 --2- 172.21.15.152:0/3723917424 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f6b20036790 0x7f6b20038c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:23.586 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.879+0000 7f6b38df6700 1 --2- 172.21.15.152:0/3723917424 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f6b20036790 0x7f6b20038c40 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f6b24009990 tx=0x7f6b24006e40 comp rx=0 tx=0).ready entity=mgr.14100 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:23.586 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:22.883+0000 7f6b29ffb700 1 -- 172.21.15.152:0/3723917424 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f6b30017080 con 0x7f6b34102910 2024-04-02T04:49:23.586 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.066+0000 7f6b3b85b700 1 -- 172.21.15.152:0/3723917424 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "config assimilate-conf"} v 0) v1 -- 0x7f6b3404bed0 con 0x7f6b34102910 2024-04-02T04:49:23.586 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.067+0000 7f6b29ffb700 1 -- 172.21.15.152:0/3723917424 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "config assimilate-conf"}]=0 v3) v1 ==== 70+0+373 (secure 0 0 0) 0x7f6b30020020 con 0x7f6b34102910 2024-04-02T04:49:23.586 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.070+0000 7f6b3b85b700 1 -- 172.21.15.152:0/3723917424 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f6b20036790 msgr2=0x7f6b20038c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:23.586 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.070+0000 7f6b3b85b700 1 --2- 172.21.15.152:0/3723917424 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f6b20036790 0x7f6b20038c40 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f6b24009990 tx=0x7f6b24006e40 comp rx=0 tx=0).stop 2024-04-02T04:49:23.586 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.070+0000 7f6b3b85b700 1 -- 172.21.15.152:0/3723917424 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6b34102910 msgr2=0x7f6b34111750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:23.586 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.070+0000 7f6b3b85b700 1 --2- 172.21.15.152:0/3723917424 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6b34102910 0x7f6b34111750 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f6b30012da0 tx=0x7f6b30009840 comp rx=0 tx=0).stop 2024-04-02T04:49:23.586 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.070+0000 7f6b3b85b700 1 -- 172.21.15.152:0/3723917424 shutdown_connections 2024-04-02T04:49:23.586 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.070+0000 7f6b3b85b700 1 --2- 172.21.15.152:0/3723917424 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f6b20036790 0x7f6b20038c40 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:23.586 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.070+0000 7f6b3b85b700 1 --2- 172.21.15.152:0/3723917424 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6b34102910 0x7f6b34111750 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:23.586 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.070+0000 7f6b3b85b700 1 -- 172.21.15.152:0/3723917424 >> 172.21.15.152:0/3723917424 conn(0x7f6b340fe090 msgr2=0x7f6b340fec80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:23.586 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.070+0000 7f6b3b85b700 1 -- 172.21.15.152:0/3723917424 shutdown_connections 2024-04-02T04:49:23.586 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.070+0000 7f6b3b85b700 1 -- 172.21.15.152:0/3723917424 wait complete. 2024-04-02T04:49:23.586 INFO:teuthology.orchestra.run.smithi152.stdout:Enabling cephadm module... 2024-04-02T04:49:24.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:24 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/2613607413' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-02T04:49:24.831 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.955+0000 7fbbd9f19700 1 Processor -- start 2024-04-02T04:49:24.831 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.955+0000 7fbbd9f19700 1 -- start start 2024-04-02T04:49:24.831 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.956+0000 7fbbd9f19700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbbd4102900 0x7fbbd4102d10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:24.831 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.956+0000 7fbbd9f19700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbbd4103250 con 0x7fbbd4102900 2024-04-02T04:49:24.831 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.956+0000 7fbbd37fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbbd4102900 0x7fbbd4102d10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:24.831 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.956+0000 7fbbd37fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbbd4102900 0x7fbbd4102d10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:52830/0 (socket says 172.21.15.152:52830) 2024-04-02T04:49:24.831 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.956+0000 7fbbd37fe700 1 -- 172.21.15.152:0/2878704927 learned_addr learned my addr 172.21.15.152:0/2878704927 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:24.832 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.956+0000 7fbbd37fe700 1 -- 172.21.15.152:0/2878704927 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbbd4103390 con 0x7fbbd4102900 2024-04-02T04:49:24.832 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.956+0000 7fbbd37fe700 1 --2- 172.21.15.152:0/2878704927 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbbd4102900 0x7fbbd4102d10 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fbbc4009e50 tx=0x7fbbc4009840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=81a9ee7470269a13 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:24.832 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.956+0000 7fbbd27fc700 1 -- 172.21.15.152:0/2878704927 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fbbc40092e0 con 0x7fbbd4102900 2024-04-02T04:49:24.832 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.956+0000 7fbbd27fc700 1 -- 172.21.15.152:0/2878704927 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7fbbc4012430 con 0x7fbbd4102900 2024-04-02T04:49:24.832 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.956+0000 7fbbd27fc700 1 -- 172.21.15.152:0/2878704927 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fbbc40075c0 con 0x7fbbd4102900 2024-04-02T04:49:24.832 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.957+0000 7fbbd9f19700 1 -- 172.21.15.152:0/2878704927 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbbd4102900 msgr2=0x7fbbd4102d10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:24.832 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.957+0000 7fbbd9f19700 1 --2- 172.21.15.152:0/2878704927 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbbd4102900 0x7fbbd4102d10 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fbbc4009e50 tx=0x7fbbc4009840 comp rx=0 tx=0).stop 2024-04-02T04:49:24.832 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.957+0000 7fbbd9f19700 1 -- 172.21.15.152:0/2878704927 shutdown_connections 2024-04-02T04:49:24.832 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.957+0000 7fbbd9f19700 1 --2- 172.21.15.152:0/2878704927 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbbd4102900 0x7fbbd4102d10 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:24.832 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.957+0000 7fbbd9f19700 1 -- 172.21.15.152:0/2878704927 >> 172.21.15.152:0/2878704927 conn(0x7fbbd40fe0a0 msgr2=0x7fbbd41004b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:24.832 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.957+0000 7fbbd9f19700 1 -- 172.21.15.152:0/2878704927 shutdown_connections 2024-04-02T04:49:24.832 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.957+0000 7fbbd9f19700 1 -- 172.21.15.152:0/2878704927 wait complete. 2024-04-02T04:49:24.832 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.957+0000 7fbbd9f19700 1 Processor -- start 2024-04-02T04:49:24.832 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.958+0000 7fbbd9f19700 1 -- start start 2024-04-02T04:49:24.832 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.958+0000 7fbbd9f19700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbbd4102900 0x7fbbd41116c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:24.833 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.958+0000 7fbbd9f19700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbbd4111c00 con 0x7fbbd4102900 2024-04-02T04:49:24.833 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.958+0000 7fbbd37fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbbd4102900 0x7fbbd41116c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:24.833 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.958+0000 7fbbd37fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbbd4102900 0x7fbbd41116c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:52836/0 (socket says 172.21.15.152:52836) 2024-04-02T04:49:24.833 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.958+0000 7fbbd37fe700 1 -- 172.21.15.152:0/2613607413 learned_addr learned my addr 172.21.15.152:0/2613607413 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:24.833 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.958+0000 7fbbd37fe700 1 -- 172.21.15.152:0/2613607413 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbbc4009ae0 con 0x7fbbd4102900 2024-04-02T04:49:24.833 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.958+0000 7fbbd37fe700 1 --2- 172.21.15.152:0/2613607413 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbbd4102900 0x7fbbd41116c0 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7fbbc4009fd0 tx=0x7fbbc4009840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:24.833 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.958+0000 7fbbbbfff700 1 -- 172.21.15.152:0/2613607413 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fbbc40092e0 con 0x7fbbd4102900 2024-04-02T04:49:24.833 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.958+0000 7fbbd9f19700 1 -- 172.21.15.152:0/2613607413 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbbd4111e00 con 0x7fbbd4102900 2024-04-02T04:49:24.833 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.958+0000 7fbbbbfff700 1 -- 172.21.15.152:0/2613607413 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7fbbc4007cc0 con 0x7fbbd4102900 2024-04-02T04:49:24.833 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.958+0000 7fbbbbfff700 1 -- 172.21.15.152:0/2613607413 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fbbc401c9a0 con 0x7fbbd4102900 2024-04-02T04:49:24.833 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.959+0000 7fbbd9f19700 1 -- 172.21.15.152:0/2613607413 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbbd4112970 con 0x7fbbd4102900 2024-04-02T04:49:24.833 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.959+0000 7fbbbbfff700 1 -- 172.21.15.152:0/2613607413 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 4) v1 ==== 43357+0+0 (secure 0 0 0) 0x7fbbc4019070 con 0x7fbbd4102900 2024-04-02T04:49:24.833 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.959+0000 7fbbd9f19700 1 -- 172.21.15.152:0/2613607413 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbbd405f5f0 con 0x7fbbd4102900 2024-04-02T04:49:24.833 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.960+0000 7fbbbbfff700 1 --2- 172.21.15.152:0/2613607413 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7fbbbc036740 0x7fbbbc038bf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:24.833 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.960+0000 7fbbbbfff700 1 -- 172.21.15.152:0/2613607413 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 721+0+0 (secure 0 0 0) 0x7fbbc4049cd0 con 0x7fbbd4102900 2024-04-02T04:49:24.834 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.960+0000 7fbbd2ffd700 1 --2- 172.21.15.152:0/2613607413 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7fbbbc036740 0x7fbbbc038bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:24.834 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.960+0000 7fbbd2ffd700 1 --2- 172.21.15.152:0/2613607413 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7fbbbc036740 0x7fbbbc038bf0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fbbc8009990 tx=0x7fbbc8006e40 comp rx=0 tx=0).ready entity=mgr.14100 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:24.834 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:23.964+0000 7fbbbbfff700 1 -- 172.21.15.152:0/2613607413 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fbbc401cba0 con 0x7fbbd4102900 2024-04-02T04:49:24.834 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:24.198+0000 7fbbd9f19700 1 -- 172.21.15.152:0/2613607413 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "mgr module enable", "module": "cephadm"} v 0) v1 -- 0x7fbbd4112d10 con 0x7fbbd4102900 2024-04-02T04:49:24.834 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:24.251+0000 7fbbbbfff700 1 -- 172.21.15.152:0/2613607413 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mgrmap(e 5) v1 ==== 43368+0+0 (secure 0 0 0) 0x7fbbc4022d80 con 0x7fbbd4102900 2024-04-02T04:49:24.834 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:24.252+0000 7fbbbbfff700 1 -- 172.21.15.152:0/2613607413 <== mon.0 v2:172.21.15.152:3300/0 8 ==== mon_command_ack([{"prefix": "mgr module enable", "module": "cephadm"}]=0 v5) v1 ==== 86+0+0 (secure 0 0 0) 0x7fbbc40196e0 con 0x7fbbd4102900 2024-04-02T04:49:24.834 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:24.254+0000 7fbbd9f19700 1 -- 172.21.15.152:0/2613607413 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7fbbbc036740 msgr2=0x7fbbbc038bf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:24.834 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:24.254+0000 7fbbd9f19700 1 --2- 172.21.15.152:0/2613607413 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7fbbbc036740 0x7fbbbc038bf0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7fbbc8009990 tx=0x7fbbc8006e40 comp rx=0 tx=0).stop 2024-04-02T04:49:24.834 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:24.254+0000 7fbbd9f19700 1 -- 172.21.15.152:0/2613607413 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbbd4102900 msgr2=0x7fbbd41116c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:24.834 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:24.254+0000 7fbbd9f19700 1 --2- 172.21.15.152:0/2613607413 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbbd4102900 0x7fbbd41116c0 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7fbbc4009fd0 tx=0x7fbbc4009840 comp rx=0 tx=0).stop 2024-04-02T04:49:24.834 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:24.254+0000 7fbbd9f19700 1 -- 172.21.15.152:0/2613607413 shutdown_connections 2024-04-02T04:49:24.834 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:24.254+0000 7fbbd9f19700 1 --2- 172.21.15.152:0/2613607413 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7fbbbc036740 0x7fbbbc038bf0 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:24.834 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:24.254+0000 7fbbd9f19700 1 --2- 172.21.15.152:0/2613607413 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbbd4102900 0x7fbbd41116c0 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:24.835 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:24.254+0000 7fbbd9f19700 1 -- 172.21.15.152:0/2613607413 >> 172.21.15.152:0/2613607413 conn(0x7fbbd40fe0a0 msgr2=0x7fbbd40feba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:24.835 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:24.255+0000 7fbbd9f19700 1 -- 172.21.15.152:0/2613607413 shutdown_connections 2024-04-02T04:49:24.835 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:24.255+0000 7fbbd9f19700 1 -- 172.21.15.152:0/2613607413 wait complete. 2024-04-02T04:49:25.426 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:25 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/2613607413' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-02T04:49:25.426 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:25 smithi152 ceph-mon[29073]: mgrmap e5: smithi152.jivdcl(active, since 8s) 2024-04-02T04:49:26.047 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout { 2024-04-02T04:49:26.047 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-04-02T04:49:26.047 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-02T04:49:26.047 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "active_name": "smithi152.jivdcl", 2024-04-02T04:49:26.047 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-02T04:49:26.047 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout } 2024-04-02T04:49:26.047 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.217+0000 7f599c866700 1 Processor -- start 2024-04-02T04:49:26.048 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.217+0000 7f599c866700 1 -- start start 2024-04-02T04:49:26.048 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.218+0000 7f599c866700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f5998076ad0 0x7f5998105580 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:26.048 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.218+0000 7f599c866700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5998076ee0 con 0x7f5998076ad0 2024-04-02T04:49:26.048 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.218+0000 7f599659c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f5998076ad0 0x7f5998105580 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:26.048 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.218+0000 7f599659c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f5998076ad0 0x7f5998105580 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:52850/0 (socket says 172.21.15.152:52850) 2024-04-02T04:49:26.048 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.218+0000 7f599659c700 1 -- 172.21.15.152:0/264000200 learned_addr learned my addr 172.21.15.152:0/264000200 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:26.048 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.218+0000 7f599659c700 1 -- 172.21.15.152:0/264000200 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5998105ac0 con 0x7f5998076ad0 2024-04-02T04:49:26.048 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.218+0000 7f599659c700 1 --2- 172.21.15.152:0/264000200 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f5998076ad0 0x7f5998105580 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f59880098e0 tx=0x7f5988009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=65313f36f3971af1 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:26.048 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.219+0000 7f599559a700 1 -- 172.21.15.152:0/264000200 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f598800b8f0 con 0x7f5998076ad0 2024-04-02T04:49:26.048 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.219+0000 7f599559a700 1 -- 172.21.15.152:0/264000200 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7f5988011420 con 0x7f5998076ad0 2024-04-02T04:49:26.048 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.219+0000 7f599559a700 1 -- 172.21.15.152:0/264000200 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f5988010540 con 0x7f5998076ad0 2024-04-02T04:49:26.048 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.219+0000 7f599c866700 1 -- 172.21.15.152:0/264000200 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f5998076ad0 msgr2=0x7f5998105580 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:26.048 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.219+0000 7f599c866700 1 --2- 172.21.15.152:0/264000200 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f5998076ad0 0x7f5998105580 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f59880098e0 tx=0x7f5988009250 comp rx=0 tx=0).stop 2024-04-02T04:49:26.048 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.219+0000 7f599c866700 1 -- 172.21.15.152:0/264000200 shutdown_connections 2024-04-02T04:49:26.049 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.219+0000 7f599c866700 1 --2- 172.21.15.152:0/264000200 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f5998076ad0 0x7f5998105580 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:26.049 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.219+0000 7f599c866700 1 -- 172.21.15.152:0/264000200 >> 172.21.15.152:0/264000200 conn(0x7f59980fcf70 msgr2=0x7f59980ff3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:26.049 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.219+0000 7f599c866700 1 -- 172.21.15.152:0/264000200 shutdown_connections 2024-04-02T04:49:26.049 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.219+0000 7f599c866700 1 -- 172.21.15.152:0/264000200 wait complete. 2024-04-02T04:49:26.049 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.220+0000 7f599c866700 1 Processor -- start 2024-04-02T04:49:26.049 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.220+0000 7f599c866700 1 -- start start 2024-04-02T04:49:26.049 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.220+0000 7f599c866700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f5998076ad0 0x7f5998111710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:26.049 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.220+0000 7f599c866700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5998111c50 con 0x7f5998076ad0 2024-04-02T04:49:26.049 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.220+0000 7f599659c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f5998076ad0 0x7f5998111710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:26.049 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.220+0000 7f599659c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f5998076ad0 0x7f5998111710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:52852/0 (socket says 172.21.15.152:52852) 2024-04-02T04:49:26.049 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.220+0000 7f599659c700 1 -- 172.21.15.152:0/3011902952 learned_addr learned my addr 172.21.15.152:0/3011902952 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:26.049 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.221+0000 7f599659c700 1 -- 172.21.15.152:0/3011902952 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f59880094f0 con 0x7f5998076ad0 2024-04-02T04:49:26.049 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.221+0000 7f599659c700 1 --2- 172.21.15.152:0/3011902952 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f5998076ad0 0x7f5998111710 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7f59880094c0 tx=0x7f5988009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:26.049 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.221+0000 7f5986ffd700 1 -- 172.21.15.152:0/3011902952 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f5988010800 con 0x7f5998076ad0 2024-04-02T04:49:26.049 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.221+0000 7f5986ffd700 1 -- 172.21.15.152:0/3011902952 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7f5988010df0 con 0x7f5998076ad0 2024-04-02T04:49:26.050 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.221+0000 7f599c866700 1 -- 172.21.15.152:0/3011902952 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5998111e50 con 0x7f5998076ad0 2024-04-02T04:49:26.050 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.221+0000 7f5986ffd700 1 -- 172.21.15.152:0/3011902952 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f5988024e50 con 0x7f5998076ad0 2024-04-02T04:49:26.050 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.221+0000 7f599c866700 1 -- 172.21.15.152:0/3011902952 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f59981129c0 con 0x7f5998076ad0 2024-04-02T04:49:26.050 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.222+0000 7f5986ffd700 1 -- 172.21.15.152:0/3011902952 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 5) v1 ==== 43368+0+0 (secure 0 0 0) 0x7f598801a4d0 con 0x7f5998076ad0 2024-04-02T04:49:26.050 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.222+0000 7f599c866700 1 -- 172.21.15.152:0/3011902952 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f599805f5f0 con 0x7f5998076ad0 2024-04-02T04:49:26.050 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.222+0000 7f5986ffd700 1 --2- 172.21.15.152:0/3011902952 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f5980036790 0x7f5980038c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:26.050 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.222+0000 7f5986ffd700 1 -- 172.21.15.152:0/3011902952 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 721+0+0 (secure 0 0 0) 0x7f598804a5a0 con 0x7f5998076ad0 2024-04-02T04:49:26.050 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.222+0000 7f5995d9b700 1 -- 172.21.15.152:0/3011902952 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f5980036790 msgr2=0x7f5980038c40 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.152:6800/2913738627 2024-04-02T04:49:26.050 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.222+0000 7f5995d9b700 1 --2- 172.21.15.152:0/3011902952 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f5980036790 0x7f5980038c40 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-02T04:49:26.050 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.227+0000 7f5986ffd700 1 -- 172.21.15.152:0/3011902952 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f59880246d0 con 0x7f5998076ad0 2024-04-02T04:49:26.050 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.423+0000 7f5995d9b700 1 -- 172.21.15.152:0/3011902952 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f5980036790 msgr2=0x7f5980038c40 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.152:6800/2913738627 2024-04-02T04:49:26.050 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.423+0000 7f5995d9b700 1 --2- 172.21.15.152:0/3011902952 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f5980036790 0x7f5980038c40 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-04-02T04:49:26.050 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.474+0000 7f599c866700 1 -- 172.21.15.152:0/3011902952 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "mgr stat"} v 0) v1 -- 0x7f5998112d00 con 0x7f5998076ad0 2024-04-02T04:49:26.050 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.474+0000 7f5986ffd700 1 -- 172.21.15.152:0/3011902952 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "mgr stat"}]=0 v5) v1 ==== 56+0+103 (secure 0 0 0) 0x7f59880233a0 con 0x7f5998076ad0 2024-04-02T04:49:26.051 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.476+0000 7f599c866700 1 -- 172.21.15.152:0/3011902952 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f5980036790 msgr2=0x7f5980038c40 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-02T04:49:26.051 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.476+0000 7f599c866700 1 --2- 172.21.15.152:0/3011902952 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f5980036790 0x7f5980038c40 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:26.051 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.476+0000 7f599c866700 1 -- 172.21.15.152:0/3011902952 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f5998076ad0 msgr2=0x7f5998111710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:26.051 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.476+0000 7f599c866700 1 --2- 172.21.15.152:0/3011902952 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f5998076ad0 0x7f5998111710 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7f59880094c0 tx=0x7f5988009250 comp rx=0 tx=0).stop 2024-04-02T04:49:26.051 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.476+0000 7f599c866700 1 -- 172.21.15.152:0/3011902952 shutdown_connections 2024-04-02T04:49:26.051 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.476+0000 7f599c866700 1 --2- 172.21.15.152:0/3011902952 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f5980036790 0x7f5980038c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:26.051 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.476+0000 7f599c866700 1 --2- 172.21.15.152:0/3011902952 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f5998076ad0 0x7f5998111710 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:26.051 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.476+0000 7f599c866700 1 -- 172.21.15.152:0/3011902952 >> 172.21.15.152:0/3011902952 conn(0x7f59980fcf70 msgr2=0x7f59980fdbc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:26.051 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.477+0000 7f599c866700 1 -- 172.21.15.152:0/3011902952 shutdown_connections 2024-04-02T04:49:26.051 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:25.477+0000 7f599c866700 1 -- 172.21.15.152:0/3011902952 wait complete. 2024-04-02T04:49:26.051 INFO:teuthology.orchestra.run.smithi152.stdout:Waiting for the mgr to restart... 2024-04-02T04:49:26.051 INFO:teuthology.orchestra.run.smithi152.stdout:Waiting for mgr epoch 5... 2024-04-02T04:49:26.288 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:26 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/3011902952' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-02T04:49:31.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:30 smithi152 ceph-mon[29073]: Active manager daemon smithi152.jivdcl restarted 2024-04-02T04:49:31.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:30 smithi152 ceph-mon[29073]: Activating manager daemon smithi152.jivdcl 2024-04-02T04:49:31.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:30 smithi152 ceph-mon[29073]: osdmap e2: 0 total, 0 up, 0 in 2024-04-02T04:49:31.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:30 smithi152 ceph-mon[29073]: mgrmap e6: smithi152.jivdcl(active, starting, since 0.00350476s) 2024-04-02T04:49:31.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:30 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi152"}]: dispatch 2024-04-02T04:49:31.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:30 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mgr metadata", "who": "smithi152.jivdcl", "id": "smithi152.jivdcl"}]: dispatch 2024-04-02T04:49:31.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:30 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T04:49:31.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:30 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T04:49:31.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:30 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T04:49:31.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:30 smithi152 ceph-mon[29073]: Manager daemon smithi152.jivdcl is now available 2024-04-02T04:49:31.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:30 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' 2024-04-02T04:49:31.008 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:30 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' 2024-04-02T04:49:31.008 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:30 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:49:31.008 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:30 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:49:31.008 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:30 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:49:31.008 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:30 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi152.jivdcl/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:49:31.008 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:30 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi152.jivdcl/trash_purge_schedule"}]: dispatch 2024-04-02T04:49:32.130 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:31 smithi152 ceph-mon[29073]: Found migration_current of "None". Setting to last migration. 2024-04-02T04:49:32.130 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:31 smithi152 ceph-mon[29073]: mgrmap e7: smithi152.jivdcl(active, since 1.00535s) 2024-04-02T04:49:32.406 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout { 2024-04-02T04:49:32.406 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-04-02T04:49:32.406 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-02T04:49:32.406 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout } 2024-04-02T04:49:32.406 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.486+0000 7f2559881700 1 Processor -- start 2024-04-02T04:49:32.406 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.486+0000 7f2559881700 1 -- start start 2024-04-02T04:49:32.406 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.486+0000 7f2559881700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2554071250 0x7f2554105540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:32.406 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.486+0000 7f2559881700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2554071660 con 0x7f2554071250 2024-04-02T04:49:32.406 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.486+0000 7f2552ffd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2554071250 0x7f2554105540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:32.406 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.486+0000 7f2552ffd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2554071250 0x7f2554105540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:52858/0 (socket says 172.21.15.152:52858) 2024-04-02T04:49:32.407 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.486+0000 7f2552ffd700 1 -- 172.21.15.152:0/3156034639 learned_addr learned my addr 172.21.15.152:0/3156034639 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:32.407 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.487+0000 7f2552ffd700 1 -- 172.21.15.152:0/3156034639 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2554105a80 con 0x7f2554071250 2024-04-02T04:49:32.407 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.487+0000 7f2552ffd700 1 --2- 172.21.15.152:0/3156034639 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2554071250 0x7f2554105540 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f25480098e0 tx=0x7f2548009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=617ea597fd5d246 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:32.407 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.487+0000 7f2551ffb700 1 -- 172.21.15.152:0/3156034639 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f254800b860 con 0x7f2554071250 2024-04-02T04:49:32.407 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.487+0000 7f2551ffb700 1 -- 172.21.15.152:0/3156034639 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7f254800be50 con 0x7f2554071250 2024-04-02T04:49:32.407 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.487+0000 7f2551ffb700 1 -- 172.21.15.152:0/3156034639 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f2548010380 con 0x7f2554071250 2024-04-02T04:49:32.407 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.488+0000 7f2559881700 1 -- 172.21.15.152:0/3156034639 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2554071250 msgr2=0x7f2554105540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:32.407 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.488+0000 7f2559881700 1 --2- 172.21.15.152:0/3156034639 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2554071250 0x7f2554105540 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 crypto rx=0x7f25480098e0 tx=0x7f2548009250 comp rx=0 tx=0).stop 2024-04-02T04:49:32.407 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.488+0000 7f2559881700 1 -- 172.21.15.152:0/3156034639 shutdown_connections 2024-04-02T04:49:32.407 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.488+0000 7f2559881700 1 --2- 172.21.15.152:0/3156034639 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2554071250 0x7f2554105540 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:32.407 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.488+0000 7f2559881700 1 -- 172.21.15.152:0/3156034639 >> 172.21.15.152:0/3156034639 conn(0x7f25540fcfb0 msgr2=0x7f25540ff3c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:32.407 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.488+0000 7f2559881700 1 -- 172.21.15.152:0/3156034639 shutdown_connections 2024-04-02T04:49:32.407 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.488+0000 7f2559881700 1 -- 172.21.15.152:0/3156034639 wait complete. 2024-04-02T04:49:32.407 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.488+0000 7f2559881700 1 Processor -- start 2024-04-02T04:49:32.408 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.488+0000 7f2559881700 1 -- start start 2024-04-02T04:49:32.408 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.488+0000 7f2559881700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2554071250 0x7f2554111760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:32.408 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.489+0000 7f2559881700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2554111ca0 con 0x7f2554071250 2024-04-02T04:49:32.408 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.489+0000 7f2552ffd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2554071250 0x7f2554111760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:32.408 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.489+0000 7f2552ffd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2554071250 0x7f2554111760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:52874/0 (socket says 172.21.15.152:52874) 2024-04-02T04:49:32.408 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.489+0000 7f2552ffd700 1 -- 172.21.15.152:0/4184633535 learned_addr learned my addr 172.21.15.152:0/4184633535 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:32.408 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.489+0000 7f2552ffd700 1 -- 172.21.15.152:0/4184633535 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f25480094f0 con 0x7f2554071250 2024-04-02T04:49:32.408 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.489+0000 7f2552ffd700 1 --2- 172.21.15.152:0/4184633535 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2554071250 0x7f2554111760 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f2548006fd0 tx=0x7f2548009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:32.408 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.489+0000 7f253b7fe700 1 -- 172.21.15.152:0/4184633535 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f2548010760 con 0x7f2554071250 2024-04-02T04:49:32.408 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.489+0000 7f2559881700 1 -- 172.21.15.152:0/4184633535 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2554111ea0 con 0x7f2554071250 2024-04-02T04:49:32.408 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.489+0000 7f253b7fe700 1 -- 172.21.15.152:0/4184633535 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7f2548010d50 con 0x7f2554071250 2024-04-02T04:49:32.408 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.489+0000 7f253b7fe700 1 -- 172.21.15.152:0/4184633535 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f254801adc0 con 0x7f2554071250 2024-04-02T04:49:32.408 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.489+0000 7f2559881700 1 -- 172.21.15.152:0/4184633535 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2554112a10 con 0x7f2554071250 2024-04-02T04:49:32.408 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.490+0000 7f253b7fe700 1 -- 172.21.15.152:0/4184633535 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 5) v1 ==== 43368+0+0 (secure 0 0 0) 0x7f25480193c0 con 0x7f2554071250 2024-04-02T04:49:32.408 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.490+0000 7f253b7fe700 1 --2- 172.21.15.152:0/4184633535 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f253c036790 0x7f253c038c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:32.409 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.490+0000 7f2559881700 1 -- 172.21.15.152:0/4184633535 --> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x7f255405f5f0 con 0x7f253c036790 2024-04-02T04:49:32.409 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.490+0000 7f253b7fe700 1 -- 172.21.15.152:0/4184633535 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 721+0+0 (secure 0 0 0) 0x7f254804ae50 con 0x7f2554071250 2024-04-02T04:49:32.409 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.490+0000 7f25527fc700 1 -- 172.21.15.152:0/4184633535 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f253c036790 msgr2=0x7f253c038c40 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.152:6800/2913738627 2024-04-02T04:49:32.409 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.490+0000 7f25527fc700 1 --2- 172.21.15.152:0/4184633535 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f253c036790 0x7f253c038c40 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-02T04:49:32.409 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.691+0000 7f25527fc700 1 -- 172.21.15.152:0/4184633535 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f253c036790 msgr2=0x7f253c038c40 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.152:6800/2913738627 2024-04-02T04:49:32.409 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:26.691+0000 7f25527fc700 1 --2- 172.21.15.152:0/4184633535 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f253c036790 0x7f253c038c40 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-04-02T04:49:32.409 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:27.092+0000 7f25527fc700 1 -- 172.21.15.152:0/4184633535 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f253c036790 msgr2=0x7f253c038c40 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.152:6800/2913738627 2024-04-02T04:49:32.409 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:27.092+0000 7f25527fc700 1 --2- 172.21.15.152:0/4184633535 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f253c036790 0x7f253c038c40 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.800000 2024-04-02T04:49:32.409 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:27.893+0000 7f25527fc700 1 -- 172.21.15.152:0/4184633535 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f253c036790 msgr2=0x7f253c038c40 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.152:6800/2913738627 2024-04-02T04:49:32.409 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:27.893+0000 7f25527fc700 1 --2- 172.21.15.152:0/4184633535 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f253c036790 0x7f253c038c40 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 1.600000 2024-04-02T04:49:32.409 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:29.494+0000 7f25527fc700 1 -- 172.21.15.152:0/4184633535 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f253c036790 msgr2=0x7f253c038c40 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.152:6800/2913738627 2024-04-02T04:49:32.409 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:29.494+0000 7f25527fc700 1 --2- 172.21.15.152:0/4184633535 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f253c036790 0x7f253c038c40 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 3.200000 2024-04-02T04:49:32.409 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:30.821+0000 7f253b7fe700 1 -- 172.21.15.152:0/4184633535 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mgrmap(e 6) v1 ==== 43135+0+0 (secure 0 0 0) 0x7f254802c350 con 0x7f2554071250 2024-04-02T04:49:32.410 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:30.821+0000 7f253b7fe700 1 -- 172.21.15.152:0/4184633535 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f253c036790 msgr2=0x7f253c038c40 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-02T04:49:32.410 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:30.821+0000 7f253b7fe700 1 --2- 172.21.15.152:0/4184633535 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f253c036790 0x7f253c038c40 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:32.410 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:31.822+0000 7f253b7fe700 1 -- 172.21.15.152:0/4184633535 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mgrmap(e 7) v1 ==== 43262+0+0 (secure 0 0 0) 0x7f254802ad20 con 0x7f2554071250 2024-04-02T04:49:32.410 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:31.822+0000 7f253b7fe700 1 --2- 172.21.15.152:0/4184633535 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f253c039f50 0x7f253c03c330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:32.410 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:31.822+0000 7f253b7fe700 1 -- 172.21.15.152:0/4184633535 --> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x7f255405f5f0 con 0x7f253c039f50 2024-04-02T04:49:32.410 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:31.823+0000 7f25527fc700 1 --2- 172.21.15.152:0/4184633535 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f253c039f50 0x7f253c03c330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:32.410 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:31.823+0000 7f25527fc700 1 --2- 172.21.15.152:0/4184633535 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f253c039f50 0x7f253c03c330 secure :-1 s=READY pgs=3 cs=0 l=1 rev1=1 crypto rx=0x7f25440039d0 tx=0x7f2544009350 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:32.410 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:31.824+0000 7f253b7fe700 1 -- 172.21.15.152:0/4184633535 <== mgr.14118 v2:172.21.15.152:6800/3544077480 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+6549 (secure 0 0 0) 0x7f255405f5f0 con 0x7f253c039f50 2024-04-02T04:49:32.410 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:31.830+0000 7f2559881700 1 -- 172.21.15.152:0/4184633535 --> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] -- command(tid 1: {"prefix": "mgr_status"}) v1 -- 0x7f2554112330 con 0x7f253c039f50 2024-04-02T04:49:32.410 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:31.830+0000 7f253b7fe700 1 -- 172.21.15.152:0/4184633535 <== mgr.14118 v2:172.21.15.152:6800/3544077480 2 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+51 (secure 0 0 0) 0x7f2554112330 con 0x7f253c039f50 2024-04-02T04:49:32.410 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:31.830+0000 7f2559881700 1 -- 172.21.15.152:0/4184633535 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f253c039f50 msgr2=0x7f253c03c330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:32.410 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:31.830+0000 7f2559881700 1 --2- 172.21.15.152:0/4184633535 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f253c039f50 0x7f253c03c330 secure :-1 s=READY pgs=3 cs=0 l=1 rev1=1 crypto rx=0x7f25440039d0 tx=0x7f2544009350 comp rx=0 tx=0).stop 2024-04-02T04:49:32.410 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:31.830+0000 7f2559881700 1 -- 172.21.15.152:0/4184633535 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2554071250 msgr2=0x7f2554111760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:32.410 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:31.830+0000 7f2559881700 1 --2- 172.21.15.152:0/4184633535 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2554071250 0x7f2554111760 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 crypto rx=0x7f2548006fd0 tx=0x7f2548009250 comp rx=0 tx=0).stop 2024-04-02T04:49:32.410 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:31.830+0000 7f2559881700 1 -- 172.21.15.152:0/4184633535 shutdown_connections 2024-04-02T04:49:32.411 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:31.830+0000 7f2559881700 1 --2- 172.21.15.152:0/4184633535 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f253c039f50 0x7f253c03c330 unknown :-1 s=CLOSED pgs=3 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:32.411 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:31.830+0000 7f2559881700 1 --2- 172.21.15.152:0/4184633535 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2554071250 0x7f2554111760 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:32.411 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:31.830+0000 7f2559881700 1 --2- 172.21.15.152:0/4184633535 >> [v2:172.21.15.152:6800/2913738627,v1:172.21.15.152:6801/2913738627] conn(0x7f253c036790 0x7f253c038c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:32.411 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:31.830+0000 7f2559881700 1 -- 172.21.15.152:0/4184633535 >> 172.21.15.152:0/4184633535 conn(0x7f25540fcfb0 msgr2=0x7f25540fdbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:32.411 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:31.830+0000 7f2559881700 1 -- 172.21.15.152:0/4184633535 shutdown_connections 2024-04-02T04:49:32.411 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:31.830+0000 7f2559881700 1 -- 172.21.15.152:0/4184633535 wait complete. 2024-04-02T04:49:32.411 INFO:teuthology.orchestra.run.smithi152.stdout:mgr epoch 5 is available 2024-04-02T04:49:32.411 INFO:teuthology.orchestra.run.smithi152.stdout:Setting orchestrator backend to cephadm... 2024-04-02T04:49:33.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:32 smithi152 ceph-mon[29073]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-02T04:49:33.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:32 smithi152 ceph-mon[29073]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-02T04:49:33.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:32 smithi152 ceph-mon[29073]: [02/Apr/2024:04:49:31] ENGINE Bus STARTING 2024-04-02T04:49:33.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:32 smithi152 ceph-mon[29073]: [02/Apr/2024:04:49:31] ENGINE Serving on https://172.21.15.152:7150 2024-04-02T04:49:33.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:32 smithi152 ceph-mon[29073]: [02/Apr/2024:04:49:31] ENGINE Bus STARTED 2024-04-02T04:49:33.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:32 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' 2024-04-02T04:49:33.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:32 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:49:33.618 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.842+0000 7f6e9896c700 1 Processor -- start 2024-04-02T04:49:33.618 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.842+0000 7f6e9896c700 1 -- start start 2024-04-02T04:49:33.618 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.843+0000 7f6e9896c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6e94102930 0x7f6e94102d40 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:33.619 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.843+0000 7f6e9896c700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6e94103280 con 0x7f6e94102930 2024-04-02T04:49:33.619 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.843+0000 7f6e9259c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6e94102930 0x7f6e94102d40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:33.619 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.843+0000 7f6e9259c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6e94102930 0x7f6e94102d40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:60490/0 (socket says 172.21.15.152:60490) 2024-04-02T04:49:33.619 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.843+0000 7f6e9259c700 1 -- 172.21.15.152:0/2241863664 learned_addr learned my addr 172.21.15.152:0/2241863664 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:33.619 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.843+0000 7f6e9259c700 1 -- 172.21.15.152:0/2241863664 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6e941033c0 con 0x7f6e94102930 2024-04-02T04:49:33.619 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.843+0000 7f6e9259c700 1 --2- 172.21.15.152:0/2241863664 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6e94102930 0x7f6e94102d40 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7f6e840098e0 tx=0x7f6e84009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=b845b0e9edafabd9 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:33.619 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.843+0000 7f6e9159a700 1 -- 172.21.15.152:0/2241863664 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6e8400b860 con 0x7f6e94102930 2024-04-02T04:49:33.619 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.843+0000 7f6e9159a700 1 -- 172.21.15.152:0/2241863664 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7f6e8400be50 con 0x7f6e94102930 2024-04-02T04:49:33.619 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.843+0000 7f6e9159a700 1 -- 172.21.15.152:0/2241863664 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6e84010380 con 0x7f6e94102930 2024-04-02T04:49:33.619 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.844+0000 7f6e9896c700 1 -- 172.21.15.152:0/2241863664 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6e94102930 msgr2=0x7f6e94102d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:33.619 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.844+0000 7f6e9896c700 1 --2- 172.21.15.152:0/2241863664 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6e94102930 0x7f6e94102d40 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 crypto rx=0x7f6e840098e0 tx=0x7f6e84009250 comp rx=0 tx=0).stop 2024-04-02T04:49:33.620 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.844+0000 7f6e9896c700 1 -- 172.21.15.152:0/2241863664 shutdown_connections 2024-04-02T04:49:33.620 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.844+0000 7f6e9896c700 1 --2- 172.21.15.152:0/2241863664 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6e94102930 0x7f6e94102d40 unknown :-1 s=CLOSED pgs=35 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:33.620 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.844+0000 7f6e9896c700 1 -- 172.21.15.152:0/2241863664 >> 172.21.15.152:0/2241863664 conn(0x7f6e940fe090 msgr2=0x7f6e941004c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:33.620 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.844+0000 7f6e9896c700 1 -- 172.21.15.152:0/2241863664 shutdown_connections 2024-04-02T04:49:33.620 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.844+0000 7f6e9896c700 1 -- 172.21.15.152:0/2241863664 wait complete. 2024-04-02T04:49:33.620 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.845+0000 7f6e9896c700 1 Processor -- start 2024-04-02T04:49:33.620 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.845+0000 7f6e9896c700 1 -- start start 2024-04-02T04:49:33.620 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.845+0000 7f6e9896c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6e94102930 0x7f6e94077b30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:33.620 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.845+0000 7f6e9896c700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6e94078070 con 0x7f6e94102930 2024-04-02T04:49:33.620 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.845+0000 7f6e9259c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6e94102930 0x7f6e94077b30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:33.620 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.845+0000 7f6e9259c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6e94102930 0x7f6e94077b30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:60502/0 (socket says 172.21.15.152:60502) 2024-04-02T04:49:33.620 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.845+0000 7f6e9259c700 1 -- 172.21.15.152:0/1857187794 learned_addr learned my addr 172.21.15.152:0/1857187794 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:33.620 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.845+0000 7f6e9259c700 1 -- 172.21.15.152:0/1857187794 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6e840094f0 con 0x7f6e94102930 2024-04-02T04:49:33.620 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.845+0000 7f6e9259c700 1 --2- 172.21.15.152:0/1857187794 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6e94102930 0x7f6e94077b30 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7f6e8400ba90 tx=0x7f6e84009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:33.620 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.845+0000 7f6e8affd700 1 -- 172.21.15.152:0/1857187794 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6e840107c0 con 0x7f6e94102930 2024-04-02T04:49:33.621 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.845+0000 7f6e8affd700 1 -- 172.21.15.152:0/1857187794 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7f6e84010db0 con 0x7f6e94102930 2024-04-02T04:49:33.621 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.845+0000 7f6e9896c700 1 -- 172.21.15.152:0/1857187794 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6e940760e0 con 0x7f6e94102930 2024-04-02T04:49:33.621 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.845+0000 7f6e8affd700 1 -- 172.21.15.152:0/1857187794 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6e8401add0 con 0x7f6e94102930 2024-04-02T04:49:33.621 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.846+0000 7f6e9896c700 1 -- 172.21.15.152:0/1857187794 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6e940765d0 con 0x7f6e94102930 2024-04-02T04:49:33.621 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.846+0000 7f6e8affd700 1 -- 172.21.15.152:0/1857187794 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 7) v1 ==== 43262+0+0 (secure 0 0 0) 0x7f6e84010920 con 0x7f6e94102930 2024-04-02T04:49:33.621 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.846+0000 7f6e9896c700 1 -- 172.21.15.152:0/1857187794 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6e9404bed0 con 0x7f6e94102930 2024-04-02T04:49:33.621 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.846+0000 7f6e8affd700 1 --2- 172.21.15.152:0/1857187794 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f6e80036660 0x7f6e80038b10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:33.621 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.846+0000 7f6e8affd700 1 -- 172.21.15.152:0/1857187794 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 936+0+0 (secure 0 0 0) 0x7f6e8404a420 con 0x7f6e94102930 2024-04-02T04:49:33.621 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.847+0000 7f6e91d9b700 1 --2- 172.21.15.152:0/1857187794 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f6e80036660 0x7f6e80038b10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:33.621 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.847+0000 7f6e91d9b700 1 --2- 172.21.15.152:0/1857187794 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f6e80036660 0x7f6e80038b10 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f6e7c009990 tx=0x7f6e7c006e40 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:33.621 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.851+0000 7f6e8affd700 1 -- 172.21.15.152:0/1857187794 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f6e8402e440 con 0x7f6e94102930 2024-04-02T04:49:33.621 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:32.986+0000 7f6e8affd700 1 -- 172.21.15.152:0/1857187794 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mgrmap(e 8) v1 ==== 43368+0+0 (secure 0 0 0) 0x7f6e8402e9c0 con 0x7f6e94102930 2024-04-02T04:49:33.621 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:33.050+0000 7f6e9896c700 1 -- 172.21.15.152:0/1857187794 --> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] -- mgr_command(tid 0: {"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}) v1 -- 0x7f6e94076ce0 con 0x7f6e80036660 2024-04-02T04:49:33.621 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:33.058+0000 7f6e8affd700 1 -- 172.21.15.152:0/1857187794 <== mgr.14118 v2:172.21.15.152:6800/3544077480 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+0 (secure 0 0 0) 0x7f6e94076ce0 con 0x7f6e80036660 2024-04-02T04:49:33.622 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:33.060+0000 7f6e9896c700 1 -- 172.21.15.152:0/1857187794 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f6e80036660 msgr2=0x7f6e80038b10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:33.622 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:33.060+0000 7f6e9896c700 1 --2- 172.21.15.152:0/1857187794 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f6e80036660 0x7f6e80038b10 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f6e7c009990 tx=0x7f6e7c006e40 comp rx=0 tx=0).stop 2024-04-02T04:49:33.622 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:33.060+0000 7f6e9896c700 1 -- 172.21.15.152:0/1857187794 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6e94102930 msgr2=0x7f6e94077b30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:33.622 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:33.060+0000 7f6e9896c700 1 --2- 172.21.15.152:0/1857187794 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6e94102930 0x7f6e94077b30 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7f6e8400ba90 tx=0x7f6e84009250 comp rx=0 tx=0).stop 2024-04-02T04:49:33.622 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:33.060+0000 7f6e9896c700 1 -- 172.21.15.152:0/1857187794 shutdown_connections 2024-04-02T04:49:33.622 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:33.060+0000 7f6e9896c700 1 --2- 172.21.15.152:0/1857187794 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f6e80036660 0x7f6e80038b10 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:33.622 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:33.060+0000 7f6e9896c700 1 --2- 172.21.15.152:0/1857187794 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f6e94102930 0x7f6e94077b30 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:33.622 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:33.060+0000 7f6e9896c700 1 -- 172.21.15.152:0/1857187794 >> 172.21.15.152:0/1857187794 conn(0x7f6e940fe090 msgr2=0x7f6e940fec10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:33.622 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:33.061+0000 7f6e9896c700 1 -- 172.21.15.152:0/1857187794 shutdown_connections 2024-04-02T04:49:33.622 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:33.061+0000 7f6e9896c700 1 -- 172.21.15.152:0/1857187794 wait complete. 2024-04-02T04:49:34.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:33 smithi152 ceph-mon[29073]: mgrmap e8: smithi152.jivdcl(active, since 2s) 2024-04-02T04:49:34.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:33 smithi152 ceph-mon[29073]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:49:34.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:33 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' 2024-04-02T04:49:34.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:33 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:49:34.743 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout value unchanged 2024-04-02T04:49:34.743 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.016+0000 7f890d7f2700 1 Processor -- start 2024-04-02T04:49:34.743 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.016+0000 7f890d7f2700 1 -- start start 2024-04-02T04:49:34.743 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.016+0000 7f890d7f2700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8908102f10 0x7f8908103320 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:34.743 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.016+0000 7f890d7f2700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8908070db0 con 0x7f8908102f10 2024-04-02T04:49:34.743 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.016+0000 7f8906ffd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8908102f10 0x7f8908103320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:34.743 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.016+0000 7f8906ffd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8908102f10 0x7f8908103320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:60510/0 (socket says 172.21.15.152:60510) 2024-04-02T04:49:34.743 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.016+0000 7f8906ffd700 1 -- 172.21.15.152:0/3016628994 learned_addr learned my addr 172.21.15.152:0/3016628994 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:34.743 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.016+0000 7f8906ffd700 1 -- 172.21.15.152:0/3016628994 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8908103860 con 0x7f8908102f10 2024-04-02T04:49:34.743 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.016+0000 7f8906ffd700 1 --2- 172.21.15.152:0/3016628994 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8908102f10 0x7f8908103320 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f88f80098e0 tx=0x7f88f8009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=29afc43cfe3a9eb1 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:34.744 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.016+0000 7f8905ffb700 1 -- 172.21.15.152:0/3016628994 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f88f800b860 con 0x7f8908102f10 2024-04-02T04:49:34.744 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.016+0000 7f8905ffb700 1 -- 172.21.15.152:0/3016628994 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7f88f800be50 con 0x7f8908102f10 2024-04-02T04:49:34.744 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.016+0000 7f8905ffb700 1 -- 172.21.15.152:0/3016628994 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f88f8010380 con 0x7f8908102f10 2024-04-02T04:49:34.744 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.017+0000 7f890d7f2700 1 -- 172.21.15.152:0/3016628994 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8908102f10 msgr2=0x7f8908103320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:34.744 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.017+0000 7f890d7f2700 1 --2- 172.21.15.152:0/3016628994 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8908102f10 0x7f8908103320 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 crypto rx=0x7f88f80098e0 tx=0x7f88f8009250 comp rx=0 tx=0).stop 2024-04-02T04:49:34.744 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.017+0000 7f890d7f2700 1 -- 172.21.15.152:0/3016628994 shutdown_connections 2024-04-02T04:49:34.744 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.017+0000 7f890d7f2700 1 --2- 172.21.15.152:0/3016628994 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8908102f10 0x7f8908103320 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:34.744 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.017+0000 7f890d7f2700 1 -- 172.21.15.152:0/3016628994 >> 172.21.15.152:0/3016628994 conn(0x7f89080fc780 msgr2=0x7f89080feb90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:34.744 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.017+0000 7f890d7f2700 1 -- 172.21.15.152:0/3016628994 shutdown_connections 2024-04-02T04:49:34.744 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.017+0000 7f890d7f2700 1 -- 172.21.15.152:0/3016628994 wait complete. 2024-04-02T04:49:34.744 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.018+0000 7f890d7f2700 1 Processor -- start 2024-04-02T04:49:34.744 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.018+0000 7f890d7f2700 1 -- start start 2024-04-02T04:49:34.744 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.018+0000 7f890d7f2700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8908102f10 0x7f8908113900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:34.744 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.018+0000 7f890d7f2700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8908113e40 con 0x7f8908102f10 2024-04-02T04:49:34.744 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.018+0000 7f8906ffd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8908102f10 0x7f8908113900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:34.745 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.018+0000 7f8906ffd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8908102f10 0x7f8908113900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:60522/0 (socket says 172.21.15.152:60522) 2024-04-02T04:49:34.745 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.018+0000 7f8906ffd700 1 -- 172.21.15.152:0/1506375047 learned_addr learned my addr 172.21.15.152:0/1506375047 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:34.745 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.018+0000 7f8906ffd700 1 -- 172.21.15.152:0/1506375047 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f88f80094f0 con 0x7f8908102f10 2024-04-02T04:49:34.745 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.019+0000 7f8906ffd700 1 --2- 172.21.15.152:0/1506375047 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8908102f10 0x7f8908113900 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f88f800ba90 tx=0x7f88f8009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:34.745 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.019+0000 7f88ef7fe700 1 -- 172.21.15.152:0/1506375047 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f88f80107c0 con 0x7f8908102f10 2024-04-02T04:49:34.745 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.019+0000 7f88ef7fe700 1 -- 172.21.15.152:0/1506375047 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7f88f8010db0 con 0x7f8908102f10 2024-04-02T04:49:34.745 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.019+0000 7f890d7f2700 1 -- 172.21.15.152:0/1506375047 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8908114040 con 0x7f8908102f10 2024-04-02T04:49:34.745 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.019+0000 7f88ef7fe700 1 -- 172.21.15.152:0/1506375047 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f88f801add0 con 0x7f8908102f10 2024-04-02T04:49:34.745 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.019+0000 7f890d7f2700 1 -- 172.21.15.152:0/1506375047 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8908114bb0 con 0x7f8908102f10 2024-04-02T04:49:34.745 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.019+0000 7f88ef7fe700 1 -- 172.21.15.152:0/1506375047 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 8) v1 ==== 43368+0+0 (secure 0 0 0) 0x7f88f802e440 con 0x7f8908102f10 2024-04-02T04:49:34.745 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.020+0000 7f88ef7fe700 1 --2- 172.21.15.152:0/1506375047 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f88f00367e0 0x7f88f0038c90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:34.745 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.020+0000 7f890d7f2700 1 -- 172.21.15.152:0/1506375047 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f890805f5f0 con 0x7f8908102f10 2024-04-02T04:49:34.745 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.020+0000 7f88ef7fe700 1 -- 172.21.15.152:0/1506375047 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 936+0+0 (secure 0 0 0) 0x7f88f804a610 con 0x7f8908102f10 2024-04-02T04:49:34.745 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.020+0000 7f89067fc700 1 --2- 172.21.15.152:0/1506375047 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f88f00367e0 0x7f88f0038c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:34.746 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.020+0000 7f89067fc700 1 --2- 172.21.15.152:0/1506375047 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f88f00367e0 0x7f88f0038c90 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f88fc009990 tx=0x7f88fc006e40 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:34.746 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.024+0000 7f88ef7fe700 1 -- 172.21.15.152:0/1506375047 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f88f8018710 con 0x7f8908102f10 2024-04-02T04:49:34.746 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.211+0000 7f890d7f2700 1 -- 172.21.15.152:0/1506375047 --> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] -- mgr_command(tid 0: {"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}) v1 -- 0x7f89080fdaa0 con 0x7f88f00367e0 2024-04-02T04:49:34.746 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.212+0000 7f88ef7fe700 1 -- 172.21.15.152:0/1506375047 <== mgr.14118 v2:172.21.15.152:6800/3544077480 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+16 (secure 0 0 0) 0x7f89080fdaa0 con 0x7f88f00367e0 2024-04-02T04:49:34.746 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.214+0000 7f890d7f2700 1 -- 172.21.15.152:0/1506375047 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f88f00367e0 msgr2=0x7f88f0038c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:34.746 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.214+0000 7f890d7f2700 1 --2- 172.21.15.152:0/1506375047 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f88f00367e0 0x7f88f0038c90 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f88fc009990 tx=0x7f88fc006e40 comp rx=0 tx=0).stop 2024-04-02T04:49:34.746 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.214+0000 7f890d7f2700 1 -- 172.21.15.152:0/1506375047 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8908102f10 msgr2=0x7f8908113900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:34.746 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.214+0000 7f890d7f2700 1 --2- 172.21.15.152:0/1506375047 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8908102f10 0x7f8908113900 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 crypto rx=0x7f88f800ba90 tx=0x7f88f8009250 comp rx=0 tx=0).stop 2024-04-02T04:49:34.746 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.214+0000 7f890d7f2700 1 -- 172.21.15.152:0/1506375047 shutdown_connections 2024-04-02T04:49:34.746 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.214+0000 7f890d7f2700 1 --2- 172.21.15.152:0/1506375047 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f88f00367e0 0x7f88f0038c90 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:34.746 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.214+0000 7f890d7f2700 1 --2- 172.21.15.152:0/1506375047 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8908102f10 0x7f8908113900 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:34.746 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.214+0000 7f890d7f2700 1 -- 172.21.15.152:0/1506375047 >> 172.21.15.152:0/1506375047 conn(0x7f89080fc780 msgr2=0x7f89080fd370 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:34.746 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.214+0000 7f890d7f2700 1 -- 172.21.15.152:0/1506375047 shutdown_connections 2024-04-02T04:49:34.747 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:34.214+0000 7f890d7f2700 1 -- 172.21.15.152:0/1506375047 wait complete. 2024-04-02T04:49:34.747 INFO:teuthology.orchestra.run.smithi152.stdout:Generating ssh key... 2024-04-02T04:49:35.138 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:34 smithi152 ceph-mon[29073]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:49:36.382 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.164+0000 7fb6cf338700 1 Processor -- start 2024-04-02T04:49:36.382 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.164+0000 7fb6cf338700 1 -- start start 2024-04-02T04:49:36.383 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.164+0000 7fb6cf338700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb6c8102920 0x7fb6c8102d30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:36.383 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.164+0000 7fb6cf338700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb6c8103270 con 0x7fb6c8102920 2024-04-02T04:49:36.383 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.164+0000 7fb6cd0d4700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb6c8102920 0x7fb6c8102d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:36.383 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.164+0000 7fb6cd0d4700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb6c8102920 0x7fb6c8102d30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:60532/0 (socket says 172.21.15.152:60532) 2024-04-02T04:49:36.383 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.164+0000 7fb6cd0d4700 1 -- 172.21.15.152:0/3393086319 learned_addr learned my addr 172.21.15.152:0/3393086319 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:36.383 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.165+0000 7fb6cd0d4700 1 -- 172.21.15.152:0/3393086319 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb6c81033b0 con 0x7fb6c8102920 2024-04-02T04:49:36.383 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.165+0000 7fb6cd0d4700 1 --2- 172.21.15.152:0/3393086319 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb6c8102920 0x7fb6c8102d30 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7fb6c40098e0 tx=0x7fb6c4009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3035167919dc329 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:36.383 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.165+0000 7fb6bffff700 1 -- 172.21.15.152:0/3393086319 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fb6c400b860 con 0x7fb6c8102920 2024-04-02T04:49:36.383 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.165+0000 7fb6bffff700 1 -- 172.21.15.152:0/3393086319 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7fb6c400be50 con 0x7fb6c8102920 2024-04-02T04:49:36.383 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.165+0000 7fb6bffff700 1 -- 172.21.15.152:0/3393086319 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fb6c4010380 con 0x7fb6c8102920 2024-04-02T04:49:36.383 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.165+0000 7fb6cf338700 1 -- 172.21.15.152:0/3393086319 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb6c8102920 msgr2=0x7fb6c8102d30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:36.383 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.165+0000 7fb6cf338700 1 --2- 172.21.15.152:0/3393086319 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb6c8102920 0x7fb6c8102d30 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 crypto rx=0x7fb6c40098e0 tx=0x7fb6c4009250 comp rx=0 tx=0).stop 2024-04-02T04:49:36.383 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.165+0000 7fb6cf338700 1 -- 172.21.15.152:0/3393086319 shutdown_connections 2024-04-02T04:49:36.383 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.165+0000 7fb6cf338700 1 --2- 172.21.15.152:0/3393086319 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb6c8102920 0x7fb6c8102d30 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:36.384 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.165+0000 7fb6cf338700 1 -- 172.21.15.152:0/3393086319 >> 172.21.15.152:0/3393086319 conn(0x7fb6c80fe0a0 msgr2=0x7fb6c81004b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:36.384 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.166+0000 7fb6cf338700 1 -- 172.21.15.152:0/3393086319 shutdown_connections 2024-04-02T04:49:36.384 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.166+0000 7fb6cf338700 1 -- 172.21.15.152:0/3393086319 wait complete. 2024-04-02T04:49:36.384 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.166+0000 7fb6cf338700 1 Processor -- start 2024-04-02T04:49:36.384 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.166+0000 7fb6cf338700 1 -- start start 2024-04-02T04:49:36.384 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.166+0000 7fb6cf338700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb6c8102920 0x7fb6c8078c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:36.384 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.166+0000 7fb6cf338700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb6c8079180 con 0x7fb6c8102920 2024-04-02T04:49:36.384 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.166+0000 7fb6cd0d4700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb6c8102920 0x7fb6c8078c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:36.384 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.166+0000 7fb6cd0d4700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb6c8102920 0x7fb6c8078c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:60542/0 (socket says 172.21.15.152:60542) 2024-04-02T04:49:36.384 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.166+0000 7fb6cd0d4700 1 -- 172.21.15.152:0/2130079906 learned_addr learned my addr 172.21.15.152:0/2130079906 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:36.384 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.167+0000 7fb6cd0d4700 1 -- 172.21.15.152:0/2130079906 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb6c40094f0 con 0x7fb6c8102920 2024-04-02T04:49:36.384 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.167+0000 7fb6cd0d4700 1 --2- 172.21.15.152:0/2130079906 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb6c8102920 0x7fb6c8078c40 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7fb6c4000c00 tx=0x7fb6c4009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:36.384 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.167+0000 7fb6bdffb700 1 -- 172.21.15.152:0/2130079906 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fb6c40107c0 con 0x7fb6c8102920 2024-04-02T04:49:36.384 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.167+0000 7fb6bdffb700 1 -- 172.21.15.152:0/2130079906 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7fb6c4010db0 con 0x7fb6c8102920 2024-04-02T04:49:36.384 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.167+0000 7fb6cf338700 1 -- 172.21.15.152:0/2130079906 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb6c80771f0 con 0x7fb6c8102920 2024-04-02T04:49:36.384 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.167+0000 7fb6bdffb700 1 -- 172.21.15.152:0/2130079906 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fb6c401ad00 con 0x7fb6c8102920 2024-04-02T04:49:36.385 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.167+0000 7fb6cf338700 1 -- 172.21.15.152:0/2130079906 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb6c80776e0 con 0x7fb6c8102920 2024-04-02T04:49:36.385 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.168+0000 7fb6bdffb700 1 -- 172.21.15.152:0/2130079906 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 8) v1 ==== 43368+0+0 (secure 0 0 0) 0x7fb6c40193c0 con 0x7fb6c8102920 2024-04-02T04:49:36.385 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.168+0000 7fb6bdffb700 1 --2- 172.21.15.152:0/2130079906 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fb6b40367e0 0x7fb6b4038c90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:36.385 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.168+0000 7fb6cf338700 1 -- 172.21.15.152:0/2130079906 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb6c804bed0 con 0x7fb6c8102920 2024-04-02T04:49:36.385 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.168+0000 7fb6bdffb700 1 -- 172.21.15.152:0/2130079906 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 936+0+0 (secure 0 0 0) 0x7fb6c404a650 con 0x7fb6c8102920 2024-04-02T04:49:36.385 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.168+0000 7fb6cc8d3700 1 --2- 172.21.15.152:0/2130079906 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fb6b40367e0 0x7fb6b4038c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:36.385 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.169+0000 7fb6cc8d3700 1 --2- 172.21.15.152:0/2130079906 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fb6b40367e0 0x7fb6b4038c90 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7fb6b8009990 tx=0x7fb6b8006e40 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:36.385 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.173+0000 7fb6bdffb700 1 -- 172.21.15.152:0/2130079906 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fb6c4022930 con 0x7fb6c8102920 2024-04-02T04:49:36.385 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.360+0000 7fb6cf338700 1 -- 172.21.15.152:0/2130079906 --> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] -- mgr_command(tid 0: {"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}) v1 -- 0x7fb6c8077df0 con 0x7fb6b40367e0 2024-04-02T04:49:36.385 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.840+0000 7fb6bdffb700 1 -- 172.21.15.152:0/2130079906 <== mgr.14118 v2:172.21.15.152:6800/3544077480 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+0 (secure 0 0 0) 0x7fb6c8077df0 con 0x7fb6b40367e0 2024-04-02T04:49:36.385 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.842+0000 7fb6cf338700 1 -- 172.21.15.152:0/2130079906 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fb6b40367e0 msgr2=0x7fb6b4038c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:36.385 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.842+0000 7fb6cf338700 1 --2- 172.21.15.152:0/2130079906 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fb6b40367e0 0x7fb6b4038c90 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7fb6b8009990 tx=0x7fb6b8006e40 comp rx=0 tx=0).stop 2024-04-02T04:49:36.385 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.842+0000 7fb6cf338700 1 -- 172.21.15.152:0/2130079906 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb6c8102920 msgr2=0x7fb6c8078c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:36.385 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.842+0000 7fb6cf338700 1 --2- 172.21.15.152:0/2130079906 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb6c8102920 0x7fb6c8078c40 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 crypto rx=0x7fb6c4000c00 tx=0x7fb6c4009250 comp rx=0 tx=0).stop 2024-04-02T04:49:36.385 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.843+0000 7fb6cf338700 1 -- 172.21.15.152:0/2130079906 shutdown_connections 2024-04-02T04:49:36.385 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.843+0000 7fb6cf338700 1 --2- 172.21.15.152:0/2130079906 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fb6b40367e0 0x7fb6b4038c90 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:36.385 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.843+0000 7fb6cf338700 1 --2- 172.21.15.152:0/2130079906 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb6c8102920 0x7fb6c8078c40 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:36.386 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.843+0000 7fb6cf338700 1 -- 172.21.15.152:0/2130079906 >> 172.21.15.152:0/2130079906 conn(0x7fb6c80fe0a0 msgr2=0x7fb6c80fec00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:36.386 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.843+0000 7fb6cf338700 1 -- 172.21.15.152:0/2130079906 shutdown_connections 2024-04-02T04:49:36.386 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:35.843+0000 7fb6cf338700 1 -- 172.21.15.152:0/2130079906 wait complete. 2024-04-02T04:49:36.948 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:36 smithi152 ceph-mon[29073]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:49:36.948 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:36 smithi152 ceph-mon[29073]: Generating ssh key... 2024-04-02T04:49:36.948 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:36 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' 2024-04-02T04:49:36.948 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:36 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' 2024-04-02T04:49:37.515 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDqUGpvT48ABXhnBd4AJXZxnZ2UI42+coiT5cnqfJddP/Ccx1tK0AJA0Iep7wybj+Fi/JmuMw0nSP2YDrfrb1tOOhQzr1Z1BIoYMS5LJlMw6Xfj5wXauoAydDAc5duMr0vTy49iZmHzqvw4scsLw8xsq8Z8TimRtMf4wUaGqiS9bbQ+mq4L9TBFjtznBdMaJ/34FAINzZpatWnndVUy59Q0onixMKkalhNfNEqnR0djmp137y1xQCxZ4M+EJzTaz912zQtNO1Xi55bzWmupb5xk34JeaoSU5KwjUx9k5IgO6qlg9d8GujXFQ7U9HwyV9ShLEMiLTSaF2jJJmV3+ClX+R0V0hG5XlZiBrOxAW7idolGX3qxAk8S8kw9RLEwmd6tylzB5eGb6Z80UUlOfkYRBhdk60AJDqc8ra0sF0aK++c57DR78f4xJ4VNjGTsbk2mTIFZEtM6Vmu3NDqp9qmyHSMAzfT8s+HmtX+IwyN3hXyI+ajNeTi50dHi6seIV/us= ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4 2024-04-02T04:49:37.515 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.747+0000 7f13b8836700 1 Processor -- start 2024-04-02T04:49:37.515 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.747+0000 7f13b8836700 1 -- start start 2024-04-02T04:49:37.515 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.748+0000 7f13b8836700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f13b4101d50 0x7f13b4102160 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:37.515 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.748+0000 7f13b8836700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f13b41026a0 con 0x7f13b4101d50 2024-04-02T04:49:37.515 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.748+0000 7f13b259c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f13b4101d50 0x7f13b4102160 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:37.515 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.748+0000 7f13b259c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f13b4101d50 0x7f13b4102160 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:60554/0 (socket says 172.21.15.152:60554) 2024-04-02T04:49:37.515 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.748+0000 7f13b259c700 1 -- 172.21.15.152:0/2100283073 learned_addr learned my addr 172.21.15.152:0/2100283073 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:37.515 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.748+0000 7f13b259c700 1 -- 172.21.15.152:0/2100283073 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f13b41027e0 con 0x7f13b4101d50 2024-04-02T04:49:37.515 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.748+0000 7f13b259c700 1 --2- 172.21.15.152:0/2100283073 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f13b4101d50 0x7f13b4102160 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f13a80098e0 tx=0x7f13a8009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2667aaea267060e server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:37.516 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.749+0000 7f13b159a700 1 -- 172.21.15.152:0/2100283073 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f13a800b860 con 0x7f13b4101d50 2024-04-02T04:49:37.516 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.749+0000 7f13b159a700 1 -- 172.21.15.152:0/2100283073 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7f13a800be50 con 0x7f13b4101d50 2024-04-02T04:49:37.516 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.749+0000 7f13b159a700 1 -- 172.21.15.152:0/2100283073 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f13a8010380 con 0x7f13b4101d50 2024-04-02T04:49:37.516 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.749+0000 7f13b8836700 1 -- 172.21.15.152:0/2100283073 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f13b4101d50 msgr2=0x7f13b4102160 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:37.516 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.749+0000 7f13b8836700 1 --2- 172.21.15.152:0/2100283073 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f13b4101d50 0x7f13b4102160 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 crypto rx=0x7f13a80098e0 tx=0x7f13a8009250 comp rx=0 tx=0).stop 2024-04-02T04:49:37.516 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.749+0000 7f13b8836700 1 -- 172.21.15.152:0/2100283073 shutdown_connections 2024-04-02T04:49:37.516 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.749+0000 7f13b8836700 1 --2- 172.21.15.152:0/2100283073 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f13b4101d50 0x7f13b4102160 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:37.516 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.749+0000 7f13b8836700 1 -- 172.21.15.152:0/2100283073 >> 172.21.15.152:0/2100283073 conn(0x7f13b40fc6c0 msgr2=0x7f13b40fead0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:37.516 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.749+0000 7f13b8836700 1 -- 172.21.15.152:0/2100283073 shutdown_connections 2024-04-02T04:49:37.516 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.749+0000 7f13b8836700 1 -- 172.21.15.152:0/2100283073 wait complete. 2024-04-02T04:49:37.516 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.750+0000 7f13b8836700 1 Processor -- start 2024-04-02T04:49:37.516 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.750+0000 7f13b8836700 1 -- start start 2024-04-02T04:49:37.516 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.750+0000 7f13b8836700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f13b4101d50 0x7f13b4101550 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:37.516 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.750+0000 7f13b8836700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f13b4114400 con 0x7f13b4101d50 2024-04-02T04:49:37.516 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.750+0000 7f13b259c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f13b4101d50 0x7f13b4101550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:37.517 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.750+0000 7f13b259c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f13b4101d50 0x7f13b4101550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:60566/0 (socket says 172.21.15.152:60566) 2024-04-02T04:49:37.517 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.750+0000 7f13b259c700 1 -- 172.21.15.152:0/3358082132 learned_addr learned my addr 172.21.15.152:0/3358082132 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:37.517 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.750+0000 7f13b259c700 1 -- 172.21.15.152:0/3358082132 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f13a80094f0 con 0x7f13b4101d50 2024-04-02T04:49:37.517 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.751+0000 7f13b259c700 1 --2- 172.21.15.152:0/3358082132 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f13b4101d50 0x7f13b4101550 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f13a8006b00 tx=0x7f13a8009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:37.517 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.751+0000 7f13a2ffd700 1 -- 172.21.15.152:0/3358082132 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f13a8010760 con 0x7f13b4101d50 2024-04-02T04:49:37.517 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.751+0000 7f13a2ffd700 1 -- 172.21.15.152:0/3358082132 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7f13a8010d50 con 0x7f13b4101d50 2024-04-02T04:49:37.517 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.751+0000 7f13b8836700 1 -- 172.21.15.152:0/3358082132 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f13b4114540 con 0x7f13b4101d50 2024-04-02T04:49:37.517 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.751+0000 7f13a2ffd700 1 -- 172.21.15.152:0/3358082132 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f13a801ad40 con 0x7f13b4101d50 2024-04-02T04:49:37.517 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.751+0000 7f13b8836700 1 -- 172.21.15.152:0/3358082132 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f13b41149d0 con 0x7f13b4101d50 2024-04-02T04:49:37.517 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.752+0000 7f13a2ffd700 1 -- 172.21.15.152:0/3358082132 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 8) v1 ==== 43368+0+0 (secure 0 0 0) 0x7f13a80108c0 con 0x7f13b4101d50 2024-04-02T04:49:37.517 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.752+0000 7f13b8836700 1 -- 172.21.15.152:0/3358082132 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f13b404bed0 con 0x7f13b4101d50 2024-04-02T04:49:37.517 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.752+0000 7f13a2ffd700 1 --2- 172.21.15.152:0/3358082132 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f139c036830 0x7f139c038ce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:37.517 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.752+0000 7f13a2ffd700 1 -- 172.21.15.152:0/3358082132 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 936+0+0 (secure 0 0 0) 0x7f13a804a5c0 con 0x7f13b4101d50 2024-04-02T04:49:37.518 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.752+0000 7f13b1d9b700 1 --2- 172.21.15.152:0/3358082132 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f139c036830 0x7f139c038ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:37.518 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.752+0000 7f13b1d9b700 1 --2- 172.21.15.152:0/3358082132 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f139c036830 0x7f139c038ce0 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f13a4009990 tx=0x7f13a4006e40 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:37.518 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.757+0000 7f13a2ffd700 1 -- 172.21.15.152:0/3358082132 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f13a802e7c0 con 0x7f13b4101d50 2024-04-02T04:49:37.518 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.944+0000 7f13b8836700 1 -- 172.21.15.152:0/3358082132 --> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] -- mgr_command(tid 0: {"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}) v1 -- 0x7f13b40fd950 con 0x7f139c036830 2024-04-02T04:49:37.518 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.945+0000 7f13a2ffd700 1 -- 172.21.15.152:0/3358082132 <== mgr.14118 v2:172.21.15.152:6800/3544077480 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+595 (secure 0 0 0) 0x7f13b40fd950 con 0x7f139c036830 2024-04-02T04:49:37.518 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.947+0000 7f13b8836700 1 -- 172.21.15.152:0/3358082132 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f139c036830 msgr2=0x7f139c038ce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:37.518 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.947+0000 7f13b8836700 1 --2- 172.21.15.152:0/3358082132 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f139c036830 0x7f139c038ce0 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f13a4009990 tx=0x7f13a4006e40 comp rx=0 tx=0).stop 2024-04-02T04:49:37.518 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.947+0000 7f13b8836700 1 -- 172.21.15.152:0/3358082132 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f13b4101d50 msgr2=0x7f13b4101550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:37.518 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.947+0000 7f13b8836700 1 --2- 172.21.15.152:0/3358082132 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f13b4101d50 0x7f13b4101550 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f13a8006b00 tx=0x7f13a8009250 comp rx=0 tx=0).stop 2024-04-02T04:49:37.518 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.947+0000 7f13b8836700 1 -- 172.21.15.152:0/3358082132 shutdown_connections 2024-04-02T04:49:37.518 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.947+0000 7f13b8836700 1 --2- 172.21.15.152:0/3358082132 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f139c036830 0x7f139c038ce0 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:37.518 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.947+0000 7f13b8836700 1 --2- 172.21.15.152:0/3358082132 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f13b4101d50 0x7f13b4101550 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:37.518 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.947+0000 7f13b8836700 1 -- 172.21.15.152:0/3358082132 >> 172.21.15.152:0/3358082132 conn(0x7f13b40fc6c0 msgr2=0x7f13b40fd240 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:37.518 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.947+0000 7f13b8836700 1 -- 172.21.15.152:0/3358082132 shutdown_connections 2024-04-02T04:49:37.519 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:36.947+0000 7f13b8836700 1 -- 172.21.15.152:0/3358082132 wait complete. 2024-04-02T04:49:37.519 INFO:teuthology.orchestra.run.smithi152.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-02T04:49:37.519 INFO:teuthology.orchestra.run.smithi152.stdout:Adding key to root@localhost authorized_keys... 2024-04-02T04:49:37.519 INFO:teuthology.orchestra.run.smithi152.stdout:Adding host smithi152... 2024-04-02T04:49:38.110 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:37 smithi152 ceph-mon[29073]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:49:39.064 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:38 smithi152 ceph-mon[29073]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi152", "addr": "172.21.15.152", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:49:40.096 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:39 smithi152 ceph-mon[29073]: Deploying cephadm binary to smithi152 2024-04-02T04:49:40.665 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout Added host 'smithi152' with addr '172.21.15.152' 2024-04-02T04:49:40.665 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.907+0000 7f722b137700 1 Processor -- start 2024-04-02T04:49:40.665 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.907+0000 7f722b137700 1 -- start start 2024-04-02T04:49:40.665 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.907+0000 7f722b137700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7224102ca0 0x7f72241030b0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:40.665 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.907+0000 7f722b137700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7224103680 con 0x7f7224102ca0 2024-04-02T04:49:40.665 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.907+0000 7f7228ed3700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7224102ca0 0x7f72241030b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:40.665 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.907+0000 7f7228ed3700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7224102ca0 0x7f72241030b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:60578/0 (socket says 172.21.15.152:60578) 2024-04-02T04:49:40.665 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.907+0000 7f7228ed3700 1 -- 172.21.15.152:0/2382712988 learned_addr learned my addr 172.21.15.152:0/2382712988 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:40.666 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.907+0000 7f7228ed3700 1 -- 172.21.15.152:0/2382712988 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f72241037c0 con 0x7f7224102ca0 2024-04-02T04:49:40.666 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.907+0000 7f7228ed3700 1 --2- 172.21.15.152:0/2382712988 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7224102ca0 0x7f72241030b0 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f72180098e0 tx=0x7f7218009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=84d225b5ee10b1e server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:40.666 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.908+0000 7f72237fe700 1 -- 172.21.15.152:0/2382712988 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f721800b860 con 0x7f7224102ca0 2024-04-02T04:49:40.666 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.908+0000 7f72237fe700 1 -- 172.21.15.152:0/2382712988 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7f721800be50 con 0x7f7224102ca0 2024-04-02T04:49:40.666 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.908+0000 7f72237fe700 1 -- 172.21.15.152:0/2382712988 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7218010380 con 0x7f7224102ca0 2024-04-02T04:49:40.666 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.908+0000 7f722b137700 1 -- 172.21.15.152:0/2382712988 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7224102ca0 msgr2=0x7f72241030b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:40.666 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.908+0000 7f722b137700 1 --2- 172.21.15.152:0/2382712988 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7224102ca0 0x7f72241030b0 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f72180098e0 tx=0x7f7218009250 comp rx=0 tx=0).stop 2024-04-02T04:49:40.666 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.908+0000 7f722b137700 1 -- 172.21.15.152:0/2382712988 shutdown_connections 2024-04-02T04:49:40.666 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.908+0000 7f722b137700 1 --2- 172.21.15.152:0/2382712988 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7224102ca0 0x7f72241030b0 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:40.666 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.908+0000 7f722b137700 1 -- 172.21.15.152:0/2382712988 >> 172.21.15.152:0/2382712988 conn(0x7f72240fc700 msgr2=0x7f72240feb10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:40.666 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.908+0000 7f722b137700 1 -- 172.21.15.152:0/2382712988 shutdown_connections 2024-04-02T04:49:40.666 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.908+0000 7f722b137700 1 -- 172.21.15.152:0/2382712988 wait complete. 2024-04-02T04:49:40.666 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.909+0000 7f722b137700 1 Processor -- start 2024-04-02T04:49:40.666 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.909+0000 7f722b137700 1 -- start start 2024-04-02T04:49:40.667 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.909+0000 7f722b137700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7224102ca0 0x7f72241138b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:40.667 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.909+0000 7f722b137700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7224113df0 con 0x7f7224102ca0 2024-04-02T04:49:40.667 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.909+0000 7f7228ed3700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7224102ca0 0x7f72241138b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:40.667 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.909+0000 7f7228ed3700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7224102ca0 0x7f72241138b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:60586/0 (socket says 172.21.15.152:60586) 2024-04-02T04:49:40.667 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.909+0000 7f7228ed3700 1 -- 172.21.15.152:0/3617160898 learned_addr learned my addr 172.21.15.152:0/3617160898 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:40.667 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.910+0000 7f7228ed3700 1 -- 172.21.15.152:0/3617160898 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f72180094f0 con 0x7f7224102ca0 2024-04-02T04:49:40.667 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.910+0000 7f7228ed3700 1 --2- 172.21.15.152:0/3617160898 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7224102ca0 0x7f72241138b0 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f7218000c00 tx=0x7f7218009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:40.667 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.910+0000 7f72217fa700 1 -- 172.21.15.152:0/3617160898 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f72180107c0 con 0x7f7224102ca0 2024-04-02T04:49:40.667 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.910+0000 7f722b137700 1 -- 172.21.15.152:0/3617160898 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7224113ff0 con 0x7f7224102ca0 2024-04-02T04:49:40.667 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.910+0000 7f72217fa700 1 -- 172.21.15.152:0/3617160898 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7f7218010db0 con 0x7f7224102ca0 2024-04-02T04:49:40.667 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.910+0000 7f72217fa700 1 -- 172.21.15.152:0/3617160898 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f721801ad00 con 0x7f7224102ca0 2024-04-02T04:49:40.667 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.910+0000 7f722b137700 1 -- 172.21.15.152:0/3617160898 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7224114b60 con 0x7f7224102ca0 2024-04-02T04:49:40.667 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.911+0000 7f72217fa700 1 -- 172.21.15.152:0/3617160898 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 8) v1 ==== 43368+0+0 (secure 0 0 0) 0x7f72180193c0 con 0x7f7224102ca0 2024-04-02T04:49:40.667 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.911+0000 7f72217fa700 1 --2- 172.21.15.152:0/3617160898 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f7214036790 0x7f7214038c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:40.667 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.911+0000 7f722b137700 1 -- 172.21.15.152:0/3617160898 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f722405f5f0 con 0x7f7224102ca0 2024-04-02T04:49:40.668 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.911+0000 7f72217fa700 1 -- 172.21.15.152:0/3617160898 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 936+0+0 (secure 0 0 0) 0x7f721804a7a0 con 0x7f7224102ca0 2024-04-02T04:49:40.668 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.911+0000 7f7223fff700 1 --2- 172.21.15.152:0/3617160898 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f7214036790 0x7f7214038c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:40.668 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.911+0000 7f7223fff700 1 --2- 172.21.15.152:0/3617160898 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f7214036790 0x7f7214038c40 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f7210009990 tx=0x7f7210006e40 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:40.668 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:37.915+0000 7f72217fa700 1 -- 172.21.15.152:0/3617160898 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f7218022930 con 0x7f7224102ca0 2024-04-02T04:49:40.668 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:38.106+0000 7f722b137700 1 -- 172.21.15.152:0/3617160898 --> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] -- mgr_command(tid 0: {"prefix": "orch host add", "hostname": "smithi152", "addr": "172.21.15.152", "target": ["mon-mgr", ""]}) v1 -- 0x7f72240fd9f0 con 0x7f7214036790 2024-04-02T04:49:40.668 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:40.091+0000 7f72217fa700 1 -- 172.21.15.152:0/3617160898 <== mgr.14118 v2:172.21.15.152:6800/3544077480 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+49 (secure 0 0 0) 0x7f72240fd9f0 con 0x7f7214036790 2024-04-02T04:49:40.668 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:40.094+0000 7f722b137700 1 -- 172.21.15.152:0/3617160898 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f7214036790 msgr2=0x7f7214038c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:40.668 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:40.094+0000 7f722b137700 1 --2- 172.21.15.152:0/3617160898 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f7214036790 0x7f7214038c40 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f7210009990 tx=0x7f7210006e40 comp rx=0 tx=0).stop 2024-04-02T04:49:40.668 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:40.094+0000 7f722b137700 1 -- 172.21.15.152:0/3617160898 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7224102ca0 msgr2=0x7f72241138b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:40.668 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:40.094+0000 7f722b137700 1 --2- 172.21.15.152:0/3617160898 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7224102ca0 0x7f72241138b0 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f7218000c00 tx=0x7f7218009250 comp rx=0 tx=0).stop 2024-04-02T04:49:40.668 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:40.094+0000 7f722b137700 1 -- 172.21.15.152:0/3617160898 shutdown_connections 2024-04-02T04:49:40.668 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:40.094+0000 7f722b137700 1 --2- 172.21.15.152:0/3617160898 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f7214036790 0x7f7214038c40 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:40.668 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:40.094+0000 7f722b137700 1 --2- 172.21.15.152:0/3617160898 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7224102ca0 0x7f72241138b0 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:40.668 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:40.094+0000 7f722b137700 1 -- 172.21.15.152:0/3617160898 >> 172.21.15.152:0/3617160898 conn(0x7f72240fc700 msgr2=0x7f72240fd2a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:40.668 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:40.094+0000 7f722b137700 1 -- 172.21.15.152:0/3617160898 shutdown_connections 2024-04-02T04:49:40.669 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:40.094+0000 7f722b137700 1 -- 172.21.15.152:0/3617160898 wait complete. 2024-04-02T04:49:40.669 INFO:teuthology.orchestra.run.smithi152.stdout:Deploying mon service with default placement... 2024-04-02T04:49:41.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:41 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' 2024-04-02T04:49:41.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:41 smithi152 ceph-mon[29073]: Added host smithi152 2024-04-02T04:49:41.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:41 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:49:42.564 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:42 smithi152 ceph-mon[29073]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:49:42.564 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:42 smithi152 ceph-mon[29073]: Saving service mon spec with placement count:5 2024-04-02T04:49:42.564 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:42 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' 2024-04-02T04:49:42.999 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-04-02T04:49:42.999 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.071+0000 7f1568e78700 1 Processor -- start 2024-04-02T04:49:42.999 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.071+0000 7f1568e78700 1 -- start start 2024-04-02T04:49:42.999 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.072+0000 7f1568e78700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f15641028f0 0x7f1564102d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:42.999 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.072+0000 7f1568e78700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1564103240 con 0x7f15641028f0 2024-04-02T04:49:42.999 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.072+0000 7f156259c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f15641028f0 0x7f1564102d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:42.999 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.072+0000 7f156259c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f15641028f0 0x7f1564102d00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:42176/0 (socket says 172.21.15.152:42176) 2024-04-02T04:49:42.999 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.072+0000 7f156259c700 1 -- 172.21.15.152:0/2022293877 learned_addr learned my addr 172.21.15.152:0/2022293877 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:42.999 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.072+0000 7f156259c700 1 -- 172.21.15.152:0/2022293877 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1564103380 con 0x7f15641028f0 2024-04-02T04:49:42.999 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.072+0000 7f156259c700 1 --2- 172.21.15.152:0/2022293877 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f15641028f0 0x7f1564102d00 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f15580098e0 tx=0x7f1558009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9276e9aeaff639b8 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:42.999 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.072+0000 7f156159a700 1 -- 172.21.15.152:0/2022293877 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f155800b860 con 0x7f15641028f0 2024-04-02T04:49:43.000 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.072+0000 7f156159a700 1 -- 172.21.15.152:0/2022293877 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7f155800be50 con 0x7f15641028f0 2024-04-02T04:49:43.000 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.072+0000 7f156159a700 1 -- 172.21.15.152:0/2022293877 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1558010380 con 0x7f15641028f0 2024-04-02T04:49:43.000 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.073+0000 7f1568e78700 1 -- 172.21.15.152:0/2022293877 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f15641028f0 msgr2=0x7f1564102d00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:43.000 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.073+0000 7f1568e78700 1 --2- 172.21.15.152:0/2022293877 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f15641028f0 0x7f1564102d00 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 crypto rx=0x7f15580098e0 tx=0x7f1558009250 comp rx=0 tx=0).stop 2024-04-02T04:49:43.000 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.073+0000 7f1568e78700 1 -- 172.21.15.152:0/2022293877 shutdown_connections 2024-04-02T04:49:43.000 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.073+0000 7f1568e78700 1 --2- 172.21.15.152:0/2022293877 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f15641028f0 0x7f1564102d00 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:43.000 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.073+0000 7f1568e78700 1 -- 172.21.15.152:0/2022293877 >> 172.21.15.152:0/2022293877 conn(0x7f15640fe070 msgr2=0x7f1564100480 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:43.000 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.073+0000 7f1568e78700 1 -- 172.21.15.152:0/2022293877 shutdown_connections 2024-04-02T04:49:43.000 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.073+0000 7f1568e78700 1 -- 172.21.15.152:0/2022293877 wait complete. 2024-04-02T04:49:43.000 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.074+0000 7f1568e78700 1 Processor -- start 2024-04-02T04:49:43.000 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.074+0000 7f1568e78700 1 -- start start 2024-04-02T04:49:43.000 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.074+0000 7f1568e78700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f15641028f0 0x7f1564078c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:43.001 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.074+0000 7f1568e78700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1564079180 con 0x7f15641028f0 2024-04-02T04:49:43.001 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.074+0000 7f156259c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f15641028f0 0x7f1564078c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:43.001 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.074+0000 7f156259c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f15641028f0 0x7f1564078c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:42190/0 (socket says 172.21.15.152:42190) 2024-04-02T04:49:43.001 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.074+0000 7f156259c700 1 -- 172.21.15.152:0/4182384778 learned_addr learned my addr 172.21.15.152:0/4182384778 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:43.001 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.074+0000 7f156259c700 1 -- 172.21.15.152:0/4182384778 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f15580094f0 con 0x7f15641028f0 2024-04-02T04:49:43.001 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.074+0000 7f156259c700 1 --2- 172.21.15.152:0/4182384778 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f15641028f0 0x7f1564078c40 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f155800b440 tx=0x7f1558009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:43.001 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.074+0000 7f1552ffd700 1 -- 172.21.15.152:0/4182384778 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1558010760 con 0x7f15641028f0 2024-04-02T04:49:43.001 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.074+0000 7f1552ffd700 1 -- 172.21.15.152:0/4182384778 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1044+0+0 (secure 0 0 0) 0x7f1558010d50 con 0x7f15641028f0 2024-04-02T04:49:43.001 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.074+0000 7f1568e78700 1 -- 172.21.15.152:0/4182384778 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f15640771f0 con 0x7f15641028f0 2024-04-02T04:49:43.001 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.074+0000 7f1552ffd700 1 -- 172.21.15.152:0/4182384778 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f155801ae20 con 0x7f15641028f0 2024-04-02T04:49:43.001 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.075+0000 7f1568e78700 1 -- 172.21.15.152:0/4182384778 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f15640776e0 con 0x7f15641028f0 2024-04-02T04:49:43.001 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.075+0000 7f1552ffd700 1 -- 172.21.15.152:0/4182384778 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 8) v1 ==== 43368+0+0 (secure 0 0 0) 0x7f155802e440 con 0x7f15641028f0 2024-04-02T04:49:43.001 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.075+0000 7f1568e78700 1 -- 172.21.15.152:0/4182384778 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f156404bed0 con 0x7f15641028f0 2024-04-02T04:49:43.001 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.075+0000 7f1552ffd700 1 --2- 172.21.15.152:0/4182384778 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f154c036770 0x7f154c038c20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:43.001 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.076+0000 7f1552ffd700 1 -- 172.21.15.152:0/4182384778 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 936+0+0 (secure 0 0 0) 0x7f155804a620 con 0x7f15641028f0 2024-04-02T04:49:43.001 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.076+0000 7f1561d9b700 1 --2- 172.21.15.152:0/4182384778 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f154c036770 0x7f154c038c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:43.002 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.076+0000 7f1561d9b700 1 --2- 172.21.15.152:0/4182384778 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f154c036770 0x7f154c038c20 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f1554009990 tx=0x7f1554006e40 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:43.002 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.080+0000 7f1552ffd700 1 -- 172.21.15.152:0/4182384778 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f1558018940 con 0x7f15641028f0 2024-04-02T04:49:43.002 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.280+0000 7f1568e78700 1 -- 172.21.15.152:0/4182384778 --> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}) v1 -- 0x7f1564077d50 con 0x7f154c036770 2024-04-02T04:49:43.002 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.287+0000 7f1552ffd700 1 -- 172.21.15.152:0/4182384778 <== mgr.14118 v2:172.21.15.152:6800/3544077480 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+24 (secure 0 0 0) 0x7f1564077d50 con 0x7f154c036770 2024-04-02T04:49:43.002 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.289+0000 7f1568e78700 1 -- 172.21.15.152:0/4182384778 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f154c036770 msgr2=0x7f154c038c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:43.002 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.289+0000 7f1568e78700 1 --2- 172.21.15.152:0/4182384778 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f154c036770 0x7f154c038c20 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f1554009990 tx=0x7f1554006e40 comp rx=0 tx=0).stop 2024-04-02T04:49:43.002 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.289+0000 7f1568e78700 1 -- 172.21.15.152:0/4182384778 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f15641028f0 msgr2=0x7f1564078c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:43.002 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.289+0000 7f1568e78700 1 --2- 172.21.15.152:0/4182384778 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f15641028f0 0x7f1564078c40 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 crypto rx=0x7f155800b440 tx=0x7f1558009250 comp rx=0 tx=0).stop 2024-04-02T04:49:43.002 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.289+0000 7f1568e78700 1 -- 172.21.15.152:0/4182384778 shutdown_connections 2024-04-02T04:49:43.002 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.289+0000 7f1568e78700 1 --2- 172.21.15.152:0/4182384778 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f154c036770 0x7f154c038c20 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:43.002 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.289+0000 7f1568e78700 1 --2- 172.21.15.152:0/4182384778 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f15641028f0 0x7f1564078c40 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:43.002 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.289+0000 7f1568e78700 1 -- 172.21.15.152:0/4182384778 >> 172.21.15.152:0/4182384778 conn(0x7f15640fe070 msgr2=0x7f15640feb30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:43.002 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.289+0000 7f1568e78700 1 -- 172.21.15.152:0/4182384778 shutdown_connections 2024-04-02T04:49:43.002 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:41.289+0000 7f1568e78700 1 -- 172.21.15.152:0/4182384778 wait complete. 2024-04-02T04:49:43.003 INFO:teuthology.orchestra.run.smithi152.stdout:Deploying mgr service with default placement... 2024-04-02T04:49:43.882 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:43 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' 2024-04-02T04:49:43.882 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:43 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' 2024-04-02T04:49:44.752 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:44 smithi152 ceph-mon[29073]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:49:44.752 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:44 smithi152 ceph-mon[29073]: Saving service mgr spec with placement count:2 2024-04-02T04:49:44.753 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:44 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' 2024-04-02T04:49:44.753 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:44 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' 2024-04-02T04:49:44.759 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-04-02T04:49:44.759 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.429+0000 7fab8e759700 1 Processor -- start 2024-04-02T04:49:44.760 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.429+0000 7fab8e759700 1 -- start start 2024-04-02T04:49:44.760 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.429+0000 7fab8e759700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fab88100d40 0x7fab88101150 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:44.760 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.429+0000 7fab8e759700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fab88063d30 con 0x7fab88100d40 2024-04-02T04:49:44.760 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.429+0000 7fab87fff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fab88100d40 0x7fab88101150 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:44.760 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.429+0000 7fab87fff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fab88100d40 0x7fab88101150 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:42196/0 (socket says 172.21.15.152:42196) 2024-04-02T04:49:44.760 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.429+0000 7fab87fff700 1 -- 172.21.15.152:0/3262565305 learned_addr learned my addr 172.21.15.152:0/3262565305 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:44.760 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.430+0000 7fab87fff700 1 -- 172.21.15.152:0/3262565305 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fab88101690 con 0x7fab88100d40 2024-04-02T04:49:44.760 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.430+0000 7fab87fff700 1 --2- 172.21.15.152:0/3262565305 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fab88100d40 0x7fab88101150 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7fab780098e0 tx=0x7fab78009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=59b15cdfafcfef3 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:44.760 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.430+0000 7fab86ffd700 1 -- 172.21.15.152:0/3262565305 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fab7800b860 con 0x7fab88100d40 2024-04-02T04:49:44.760 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.430+0000 7fab86ffd700 1 -- 172.21.15.152:0/3262565305 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fab7800be90 con 0x7fab88100d40 2024-04-02T04:49:44.760 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.430+0000 7fab86ffd700 1 -- 172.21.15.152:0/3262565305 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fab780103c0 con 0x7fab88100d40 2024-04-02T04:49:44.760 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.431+0000 7fab8e759700 1 -- 172.21.15.152:0/3262565305 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fab88100d40 msgr2=0x7fab88101150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:44.760 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.431+0000 7fab8e759700 1 --2- 172.21.15.152:0/3262565305 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fab88100d40 0x7fab88101150 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7fab780098e0 tx=0x7fab78009250 comp rx=0 tx=0).stop 2024-04-02T04:49:44.760 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.431+0000 7fab8e759700 1 -- 172.21.15.152:0/3262565305 shutdown_connections 2024-04-02T04:49:44.761 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.431+0000 7fab8e759700 1 --2- 172.21.15.152:0/3262565305 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fab88100d40 0x7fab88101150 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:44.761 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.431+0000 7fab8e759700 1 -- 172.21.15.152:0/3262565305 >> 172.21.15.152:0/3262565305 conn(0x7fab880fc6e0 msgr2=0x7fab880feb10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:44.761 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.431+0000 7fab8e759700 1 -- 172.21.15.152:0/3262565305 shutdown_connections 2024-04-02T04:49:44.761 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.431+0000 7fab8e759700 1 -- 172.21.15.152:0/3262565305 wait complete. 2024-04-02T04:49:44.761 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.431+0000 7fab8e759700 1 Processor -- start 2024-04-02T04:49:44.761 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.432+0000 7fab8e759700 1 -- start start 2024-04-02T04:49:44.761 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.432+0000 7fab8e759700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fab88100d40 0x7fab88103770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:44.762 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.432+0000 7fab8e759700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fab88103cb0 con 0x7fab88100d40 2024-04-02T04:49:44.762 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.432+0000 7fab87fff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fab88100d40 0x7fab88103770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:44.762 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.432+0000 7fab87fff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fab88100d40 0x7fab88103770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:42204/0 (socket says 172.21.15.152:42204) 2024-04-02T04:49:44.762 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.432+0000 7fab87fff700 1 -- 172.21.15.152:0/3352655072 learned_addr learned my addr 172.21.15.152:0/3352655072 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:44.762 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.432+0000 7fab87fff700 1 -- 172.21.15.152:0/3352655072 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fab780094f0 con 0x7fab88100d40 2024-04-02T04:49:44.763 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.432+0000 7fab87fff700 1 --2- 172.21.15.152:0/3352655072 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fab88100d40 0x7fab88103770 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7fab780090b0 tx=0x7fab78009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:44.763 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.432+0000 7fab84ff9700 1 -- 172.21.15.152:0/3352655072 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fab78010840 con 0x7fab88100d40 2024-04-02T04:49:44.763 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.433+0000 7fab8e759700 1 -- 172.21.15.152:0/3352655072 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fab88101d20 con 0x7fab88100d40 2024-04-02T04:49:44.763 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.433+0000 7fab84ff9700 1 -- 172.21.15.152:0/3352655072 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fab78010e70 con 0x7fab88100d40 2024-04-02T04:49:44.763 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.433+0000 7fab84ff9700 1 -- 172.21.15.152:0/3352655072 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fab78024e40 con 0x7fab88100d40 2024-04-02T04:49:44.763 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.433+0000 7fab8e759700 1 -- 172.21.15.152:0/3352655072 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fab88102210 con 0x7fab88100d40 2024-04-02T04:49:44.763 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.434+0000 7fab8e759700 1 -- 172.21.15.152:0/3352655072 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fab8804bed0 con 0x7fab88100d40 2024-04-02T04:49:44.764 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.434+0000 7fab84ff9700 1 -- 172.21.15.152:0/3352655072 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 8) v1 ==== 43368+0+0 (secure 0 0 0) 0x7fab7801a530 con 0x7fab88100d40 2024-04-02T04:49:44.764 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.434+0000 7fab84ff9700 1 --2- 172.21.15.152:0/3352655072 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fab70036740 0x7fab70038bf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:44.764 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.434+0000 7fab84ff9700 1 -- 172.21.15.152:0/3352655072 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 936+0+0 (secure 0 0 0) 0x7fab7804a910 con 0x7fab88100d40 2024-04-02T04:49:44.764 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.434+0000 7fab877fe700 1 --2- 172.21.15.152:0/3352655072 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fab70036740 0x7fab70038bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:44.764 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.435+0000 7fab877fe700 1 --2- 172.21.15.152:0/3352655072 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fab70036740 0x7fab70038bf0 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7fab7c009990 tx=0x7fab7c006e40 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:44.764 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.439+0000 7fab84ff9700 1 -- 172.21.15.152:0/3352655072 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fab78018940 con 0x7fab88100d40 2024-04-02T04:49:44.765 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.651+0000 7fab8e759700 1 -- 172.21.15.152:0/3352655072 --> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}) v1 -- 0x7fab88102920 con 0x7fab70036740 2024-04-02T04:49:44.765 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.655+0000 7fab84ff9700 1 -- 172.21.15.152:0/3352655072 <== mgr.14118 v2:172.21.15.152:6800/3544077480 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+24 (secure 0 0 0) 0x7fab88102920 con 0x7fab70036740 2024-04-02T04:49:44.765 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.657+0000 7fab8e759700 1 -- 172.21.15.152:0/3352655072 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fab70036740 msgr2=0x7fab70038bf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:44.765 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.657+0000 7fab8e759700 1 --2- 172.21.15.152:0/3352655072 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fab70036740 0x7fab70038bf0 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7fab7c009990 tx=0x7fab7c006e40 comp rx=0 tx=0).stop 2024-04-02T04:49:44.765 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.657+0000 7fab8e759700 1 -- 172.21.15.152:0/3352655072 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fab88100d40 msgr2=0x7fab88103770 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:44.765 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.657+0000 7fab8e759700 1 --2- 172.21.15.152:0/3352655072 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fab88100d40 0x7fab88103770 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7fab780090b0 tx=0x7fab78009250 comp rx=0 tx=0).stop 2024-04-02T04:49:44.766 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.657+0000 7fab8e759700 1 -- 172.21.15.152:0/3352655072 shutdown_connections 2024-04-02T04:49:44.766 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.657+0000 7fab8e759700 1 --2- 172.21.15.152:0/3352655072 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fab70036740 0x7fab70038bf0 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:44.766 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.657+0000 7fab8e759700 1 --2- 172.21.15.152:0/3352655072 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fab88100d40 0x7fab88103770 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:44.766 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.657+0000 7fab8e759700 1 -- 172.21.15.152:0/3352655072 >> 172.21.15.152:0/3352655072 conn(0x7fab880fc6e0 msgr2=0x7fab880fd2b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:44.766 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.658+0000 7fab8e759700 1 -- 172.21.15.152:0/3352655072 shutdown_connections 2024-04-02T04:49:44.766 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:43.658+0000 7fab8e759700 1 -- 172.21.15.152:0/3352655072 wait complete. 2024-04-02T04:49:44.766 INFO:teuthology.orchestra.run.smithi152.stdout:Deploying crash service with default placement... 2024-04-02T04:49:45.962 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout Scheduled crash update... 2024-04-02T04:49:45.962 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.164+0000 7f7c197af700 1 Processor -- start 2024-04-02T04:49:45.962 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.164+0000 7f7c197af700 1 -- start start 2024-04-02T04:49:45.962 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.164+0000 7f7c197af700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7c14078c10 0x7f7c14077210 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:45.962 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.164+0000 7f7c197af700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7c14079020 con 0x7f7c14078c10 2024-04-02T04:49:45.962 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.164+0000 7f7c12ffd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7c14078c10 0x7f7c14077210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:45.962 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.164+0000 7f7c12ffd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7c14078c10 0x7f7c14077210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:42210/0 (socket says 172.21.15.152:42210) 2024-04-02T04:49:45.963 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.164+0000 7f7c12ffd700 1 -- 172.21.15.152:0/1478605700 learned_addr learned my addr 172.21.15.152:0/1478605700 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:45.963 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.165+0000 7f7c12ffd700 1 -- 172.21.15.152:0/1478605700 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7c14077750 con 0x7f7c14078c10 2024-04-02T04:49:45.963 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.165+0000 7f7c12ffd700 1 --2- 172.21.15.152:0/1478605700 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7c14078c10 0x7f7c14077210 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f7c040098e0 tx=0x7f7c04009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=f9e2cf38cb9de3ef server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:45.963 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.165+0000 7f7c11ffb700 1 -- 172.21.15.152:0/1478605700 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7c0400b860 con 0x7f7c14078c10 2024-04-02T04:49:45.963 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.165+0000 7f7c11ffb700 1 -- 172.21.15.152:0/1478605700 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f7c0400be90 con 0x7f7c14078c10 2024-04-02T04:49:45.963 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.165+0000 7f7c11ffb700 1 -- 172.21.15.152:0/1478605700 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7c040103c0 con 0x7f7c14078c10 2024-04-02T04:49:45.963 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.165+0000 7f7c197af700 1 -- 172.21.15.152:0/1478605700 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7c14078c10 msgr2=0x7f7c14077210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:45.963 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.165+0000 7f7c197af700 1 --2- 172.21.15.152:0/1478605700 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7c14078c10 0x7f7c14077210 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f7c040098e0 tx=0x7f7c04009250 comp rx=0 tx=0).stop 2024-04-02T04:49:45.963 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.165+0000 7f7c197af700 1 -- 172.21.15.152:0/1478605700 shutdown_connections 2024-04-02T04:49:45.963 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.165+0000 7f7c197af700 1 --2- 172.21.15.152:0/1478605700 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7c14078c10 0x7f7c14077210 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:45.963 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.165+0000 7f7c197af700 1 -- 172.21.15.152:0/1478605700 >> 172.21.15.152:0/1478605700 conn(0x7f7c140fe070 msgr2=0x7f7c141004a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:45.963 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.166+0000 7f7c197af700 1 -- 172.21.15.152:0/1478605700 shutdown_connections 2024-04-02T04:49:45.963 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.166+0000 7f7c197af700 1 -- 172.21.15.152:0/1478605700 wait complete. 2024-04-02T04:49:45.963 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.166+0000 7f7c197af700 1 Processor -- start 2024-04-02T04:49:45.964 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.166+0000 7f7c197af700 1 -- start start 2024-04-02T04:49:45.964 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.166+0000 7f7c197af700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7c14078c10 0x7f7c14113940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:45.964 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.166+0000 7f7c197af700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7c14113e80 con 0x7f7c14078c10 2024-04-02T04:49:45.964 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.167+0000 7f7c12ffd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7c14078c10 0x7f7c14113940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:45.964 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.167+0000 7f7c12ffd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7c14078c10 0x7f7c14113940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:42222/0 (socket says 172.21.15.152:42222) 2024-04-02T04:49:45.964 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.167+0000 7f7c12ffd700 1 -- 172.21.15.152:0/3847988812 learned_addr learned my addr 172.21.15.152:0/3847988812 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:45.964 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.167+0000 7f7c12ffd700 1 -- 172.21.15.152:0/3847988812 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7c040094f0 con 0x7f7c14078c10 2024-04-02T04:49:45.964 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.167+0000 7f7c12ffd700 1 --2- 172.21.15.152:0/3847988812 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7c14078c10 0x7f7c14113940 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f7c040094c0 tx=0x7f7c04009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:45.964 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.167+0000 7f7bfb7fe700 1 -- 172.21.15.152:0/3847988812 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7c04010890 con 0x7f7c14078c10 2024-04-02T04:49:45.964 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.167+0000 7f7bfb7fe700 1 -- 172.21.15.152:0/3847988812 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f7c04010ec0 con 0x7f7c14078c10 2024-04-02T04:49:45.964 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.167+0000 7f7c197af700 1 -- 172.21.15.152:0/3847988812 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7c14114080 con 0x7f7c14078c10 2024-04-02T04:49:45.964 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.167+0000 7f7bfb7fe700 1 -- 172.21.15.152:0/3847988812 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7c040243f0 con 0x7f7c14078c10 2024-04-02T04:49:45.965 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.167+0000 7f7c197af700 1 -- 172.21.15.152:0/3847988812 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7c14114bf0 con 0x7f7c14078c10 2024-04-02T04:49:45.965 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.168+0000 7f7c197af700 1 -- 172.21.15.152:0/3847988812 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7c1405f5f0 con 0x7f7c14078c10 2024-04-02T04:49:45.965 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.168+0000 7f7bfb7fe700 1 -- 172.21.15.152:0/3847988812 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 8) v1 ==== 43368+0+0 (secure 0 0 0) 0x7f7c040109f0 con 0x7f7c14078c10 2024-04-02T04:49:45.965 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.168+0000 7f7bfb7fe700 1 --2- 172.21.15.152:0/3847988812 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f7bfc0367e0 0x7f7bfc038c90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:45.965 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.168+0000 7f7bfb7fe700 1 -- 172.21.15.152:0/3847988812 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 936+0+0 (secure 0 0 0) 0x7f7c0404a260 con 0x7f7c14078c10 2024-04-02T04:49:45.965 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.168+0000 7f7c127fc700 1 --2- 172.21.15.152:0/3847988812 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f7bfc0367e0 0x7f7bfc038c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:45.965 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.169+0000 7f7c127fc700 1 --2- 172.21.15.152:0/3847988812 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f7bfc0367e0 0x7f7bfc038c90 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f7c08009990 tx=0x7f7c08006e40 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:45.965 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.173+0000 7f7bfb7fe700 1 -- 172.21.15.152:0/3847988812 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f7c0402b840 con 0x7f7c14078c10 2024-04-02T04:49:45.965 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.371+0000 7f7c197af700 1 -- 172.21.15.152:0/3847988812 --> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}) v1 -- 0x7f7c140ff430 con 0x7f7bfc0367e0 2024-04-02T04:49:45.965 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.376+0000 7f7bfb7fe700 1 -- 172.21.15.152:0/3847988812 <== mgr.14118 v2:172.21.15.152:6800/3544077480 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+26 (secure 0 0 0) 0x7f7c140ff430 con 0x7f7bfc0367e0 2024-04-02T04:49:45.965 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.378+0000 7f7c197af700 1 -- 172.21.15.152:0/3847988812 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f7bfc0367e0 msgr2=0x7f7bfc038c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:45.965 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.378+0000 7f7c197af700 1 --2- 172.21.15.152:0/3847988812 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f7bfc0367e0 0x7f7bfc038c90 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f7c08009990 tx=0x7f7c08006e40 comp rx=0 tx=0).stop 2024-04-02T04:49:45.965 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.378+0000 7f7c197af700 1 -- 172.21.15.152:0/3847988812 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7c14078c10 msgr2=0x7f7c14113940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:45.965 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.378+0000 7f7c197af700 1 --2- 172.21.15.152:0/3847988812 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7c14078c10 0x7f7c14113940 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f7c040094c0 tx=0x7f7c04009250 comp rx=0 tx=0).stop 2024-04-02T04:49:45.965 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.378+0000 7f7c197af700 1 -- 172.21.15.152:0/3847988812 shutdown_connections 2024-04-02T04:49:45.966 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.378+0000 7f7c197af700 1 --2- 172.21.15.152:0/3847988812 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f7bfc0367e0 0x7f7bfc038c90 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:45.966 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.378+0000 7f7c197af700 1 --2- 172.21.15.152:0/3847988812 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7c14078c10 0x7f7c14113940 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:45.966 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.378+0000 7f7c197af700 1 -- 172.21.15.152:0/3847988812 >> 172.21.15.152:0/3847988812 conn(0x7f7c140fe070 msgr2=0x7f7c140fec60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:45.966 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.378+0000 7f7c197af700 1 -- 172.21.15.152:0/3847988812 shutdown_connections 2024-04-02T04:49:45.966 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:45.378+0000 7f7c197af700 1 -- 172.21.15.152:0/3847988812 wait complete. 2024-04-02T04:49:45.966 INFO:teuthology.orchestra.run.smithi152.stdout:Deploying prometheus service with default placement... 2024-04-02T04:49:46.634 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:46 smithi152 ceph-mon[29073]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:49:46.635 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:46 smithi152 ceph-mon[29073]: Saving service crash spec with placement * 2024-04-02T04:49:46.635 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:46 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' 2024-04-02T04:49:46.635 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:46 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' 2024-04-02T04:49:47.586 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout Scheduled prometheus update... 2024-04-02T04:49:47.586 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.411+0000 7f4b21875700 1 Processor -- start 2024-04-02T04:49:47.586 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.411+0000 7f4b21875700 1 -- start start 2024-04-02T04:49:47.586 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.411+0000 7f4b21875700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4b1c1028c0 0x7f4b1c102cd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:47.587 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.411+0000 7f4b21875700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b1c103210 con 0x7f4b1c1028c0 2024-04-02T04:49:47.587 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.411+0000 7f4b1affd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4b1c1028c0 0x7f4b1c102cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:47.587 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.412+0000 7f4b1affd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4b1c1028c0 0x7f4b1c102cd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:42232/0 (socket says 172.21.15.152:42232) 2024-04-02T04:49:47.587 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.412+0000 7f4b1affd700 1 -- 172.21.15.152:0/3633372429 learned_addr learned my addr 172.21.15.152:0/3633372429 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:47.587 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.412+0000 7f4b1affd700 1 -- 172.21.15.152:0/3633372429 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4b1c103350 con 0x7f4b1c1028c0 2024-04-02T04:49:47.587 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.412+0000 7f4b1affd700 1 --2- 172.21.15.152:0/3633372429 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4b1c1028c0 0x7f4b1c102cd0 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f4b100098e0 tx=0x7f4b10009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ad0bd2e71259ea77 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:47.587 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.412+0000 7f4b19ffb700 1 -- 172.21.15.152:0/3633372429 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f4b1000b860 con 0x7f4b1c1028c0 2024-04-02T04:49:47.587 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.412+0000 7f4b19ffb700 1 -- 172.21.15.152:0/3633372429 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f4b1000be90 con 0x7f4b1c1028c0 2024-04-02T04:49:47.587 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.412+0000 7f4b19ffb700 1 -- 172.21.15.152:0/3633372429 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f4b100103c0 con 0x7f4b1c1028c0 2024-04-02T04:49:47.587 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.413+0000 7f4b21875700 1 -- 172.21.15.152:0/3633372429 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4b1c1028c0 msgr2=0x7f4b1c102cd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:47.587 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.413+0000 7f4b21875700 1 --2- 172.21.15.152:0/3633372429 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4b1c1028c0 0x7f4b1c102cd0 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f4b100098e0 tx=0x7f4b10009250 comp rx=0 tx=0).stop 2024-04-02T04:49:47.587 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.413+0000 7f4b21875700 1 -- 172.21.15.152:0/3633372429 shutdown_connections 2024-04-02T04:49:47.587 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.413+0000 7f4b21875700 1 --2- 172.21.15.152:0/3633372429 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4b1c1028c0 0x7f4b1c102cd0 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:47.587 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.413+0000 7f4b21875700 1 -- 172.21.15.152:0/3633372429 >> 172.21.15.152:0/3633372429 conn(0x7f4b1c0fe080 msgr2=0x7f4b1c100490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:47.588 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.413+0000 7f4b21875700 1 -- 172.21.15.152:0/3633372429 shutdown_connections 2024-04-02T04:49:47.588 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.413+0000 7f4b21875700 1 -- 172.21.15.152:0/3633372429 wait complete. 2024-04-02T04:49:47.588 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.414+0000 7f4b21875700 1 Processor -- start 2024-04-02T04:49:47.588 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.414+0000 7f4b21875700 1 -- start start 2024-04-02T04:49:47.588 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.414+0000 7f4b21875700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4b1c1028c0 0x7f4b1c1116c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:47.588 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.414+0000 7f4b21875700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4b1c111c00 con 0x7f4b1c1028c0 2024-04-02T04:49:47.588 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.414+0000 7f4b1affd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4b1c1028c0 0x7f4b1c1116c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:47.588 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.414+0000 7f4b1affd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4b1c1028c0 0x7f4b1c1116c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:42248/0 (socket says 172.21.15.152:42248) 2024-04-02T04:49:47.588 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.414+0000 7f4b1affd700 1 -- 172.21.15.152:0/1818219005 learned_addr learned my addr 172.21.15.152:0/1818219005 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:47.588 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.415+0000 7f4b1affd700 1 -- 172.21.15.152:0/1818219005 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4b100094f0 con 0x7f4b1c1028c0 2024-04-02T04:49:47.588 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.415+0000 7f4b1affd700 1 --2- 172.21.15.152:0/1818219005 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4b1c1028c0 0x7f4b1c1116c0 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f4b10011680 tx=0x7f4b10009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:47.588 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.415+0000 7f4b037fe700 1 -- 172.21.15.152:0/1818219005 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f4b10010890 con 0x7f4b1c1028c0 2024-04-02T04:49:47.588 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.415+0000 7f4b037fe700 1 -- 172.21.15.152:0/1818219005 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f4b10010ec0 con 0x7f4b1c1028c0 2024-04-02T04:49:47.588 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.415+0000 7f4b21875700 1 -- 172.21.15.152:0/1818219005 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4b1c111e00 con 0x7f4b1c1028c0 2024-04-02T04:49:47.589 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.415+0000 7f4b037fe700 1 -- 172.21.15.152:0/1818219005 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f4b100243f0 con 0x7f4b1c1028c0 2024-04-02T04:49:47.589 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.415+0000 7f4b21875700 1 -- 172.21.15.152:0/1818219005 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4b1c112970 con 0x7f4b1c1028c0 2024-04-02T04:49:47.589 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.416+0000 7f4b037fe700 1 -- 172.21.15.152:0/1818219005 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 8) v1 ==== 43368+0+0 (secure 0 0 0) 0x7f4b1002d3f0 con 0x7f4b1c1028c0 2024-04-02T04:49:47.589 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.416+0000 7f4b21875700 1 -- 172.21.15.152:0/1818219005 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4b1c05f5f0 con 0x7f4b1c1028c0 2024-04-02T04:49:47.589 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.416+0000 7f4b037fe700 1 --2- 172.21.15.152:0/1818219005 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f4b04036790 0x7f4b04038c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:47.589 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.416+0000 7f4b037fe700 1 -- 172.21.15.152:0/1818219005 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 936+0+0 (secure 0 0 0) 0x7f4b1004a280 con 0x7f4b1c1028c0 2024-04-02T04:49:47.589 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.416+0000 7f4b1a7fc700 1 --2- 172.21.15.152:0/1818219005 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f4b04036790 0x7f4b04038c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:47.589 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.417+0000 7f4b1a7fc700 1 --2- 172.21.15.152:0/1818219005 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f4b04036790 0x7f4b04038c40 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f4b0c009990 tx=0x7f4b0c006e40 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:47.589 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.423+0000 7f4b037fe700 1 -- 172.21.15.152:0/1818219005 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f4b1002b8e0 con 0x7f4b1c1028c0 2024-04-02T04:49:47.589 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.630+0000 7f4b21875700 1 -- 172.21.15.152:0/1818219005 --> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}) v1 -- 0x7f4b1c0ff340 con 0x7f4b04036790 2024-04-02T04:49:47.589 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.635+0000 7f4b037fe700 1 -- 172.21.15.152:0/1818219005 <== mgr.14118 v2:172.21.15.152:6800/3544077480 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+31 (secure 0 0 0) 0x7f4b1c0ff340 con 0x7f4b04036790 2024-04-02T04:49:47.589 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.637+0000 7f4b21875700 1 -- 172.21.15.152:0/1818219005 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f4b04036790 msgr2=0x7f4b04038c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:47.589 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.637+0000 7f4b21875700 1 --2- 172.21.15.152:0/1818219005 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f4b04036790 0x7f4b04038c40 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7f4b0c009990 tx=0x7f4b0c006e40 comp rx=0 tx=0).stop 2024-04-02T04:49:47.589 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.637+0000 7f4b21875700 1 -- 172.21.15.152:0/1818219005 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4b1c1028c0 msgr2=0x7f4b1c1116c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:47.590 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.637+0000 7f4b21875700 1 --2- 172.21.15.152:0/1818219005 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4b1c1028c0 0x7f4b1c1116c0 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 crypto rx=0x7f4b10011680 tx=0x7f4b10009250 comp rx=0 tx=0).stop 2024-04-02T04:49:47.590 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.637+0000 7f4b21875700 1 -- 172.21.15.152:0/1818219005 shutdown_connections 2024-04-02T04:49:47.590 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.637+0000 7f4b21875700 1 --2- 172.21.15.152:0/1818219005 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f4b04036790 0x7f4b04038c40 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:47.590 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.637+0000 7f4b21875700 1 --2- 172.21.15.152:0/1818219005 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4b1c1028c0 0x7f4b1c1116c0 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:47.590 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.637+0000 7f4b21875700 1 -- 172.21.15.152:0/1818219005 >> 172.21.15.152:0/1818219005 conn(0x7f4b1c0fe080 msgr2=0x7f4b1c0febf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:47.590 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.637+0000 7f4b21875700 1 -- 172.21.15.152:0/1818219005 shutdown_connections 2024-04-02T04:49:47.590 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:46.637+0000 7f4b21875700 1 -- 172.21.15.152:0/1818219005 wait complete. 2024-04-02T04:49:47.590 INFO:teuthology.orchestra.run.smithi152.stdout:Deploying grafana service with default placement... 2024-04-02T04:49:47.708 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:47 smithi152 ceph-mon[29073]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:49:47.708 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:47 smithi152 ceph-mon[29073]: Saving service prometheus spec with placement count:1 2024-04-02T04:49:47.708 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:47 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' 2024-04-02T04:49:49.129 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout Scheduled grafana update... 2024-04-02T04:49:49.129 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.003+0000 7fe8567fe700 1 Processor -- start 2024-04-02T04:49:49.129 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.003+0000 7fe8567fe700 1 -- start start 2024-04-02T04:49:49.129 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.003+0000 7fe8567fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe850076c70 0x7fe850105440 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:49.129 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.003+0000 7fe8567fe700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe850105980 con 0x7fe850076c70 2024-04-02T04:49:49.129 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.003+0000 7fe84ffff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe850076c70 0x7fe850105440 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:49.130 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.003+0000 7fe84ffff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe850076c70 0x7fe850105440 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:42262/0 (socket says 172.21.15.152:42262) 2024-04-02T04:49:49.130 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.003+0000 7fe84ffff700 1 -- 172.21.15.152:0/1651085770 learned_addr learned my addr 172.21.15.152:0/1651085770 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:49.130 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.003+0000 7fe84ffff700 1 -- 172.21.15.152:0/1651085770 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe850105ac0 con 0x7fe850076c70 2024-04-02T04:49:49.130 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.003+0000 7fe84ffff700 1 --2- 172.21.15.152:0/1651085770 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe850076c70 0x7fe850105440 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7fe8400098e0 tx=0x7fe840009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c9db551995f6056a server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:49.130 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.004+0000 7fe84effd700 1 -- 172.21.15.152:0/1651085770 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fe84000b860 con 0x7fe850076c70 2024-04-02T04:49:49.130 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.004+0000 7fe84effd700 1 -- 172.21.15.152:0/1651085770 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fe84000be90 con 0x7fe850076c70 2024-04-02T04:49:49.130 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.004+0000 7fe84effd700 1 -- 172.21.15.152:0/1651085770 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fe8400103c0 con 0x7fe850076c70 2024-04-02T04:49:49.130 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.004+0000 7fe8567fe700 1 -- 172.21.15.152:0/1651085770 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe850076c70 msgr2=0x7fe850105440 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:49.130 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.004+0000 7fe8567fe700 1 --2- 172.21.15.152:0/1651085770 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe850076c70 0x7fe850105440 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 crypto rx=0x7fe8400098e0 tx=0x7fe840009250 comp rx=0 tx=0).stop 2024-04-02T04:49:49.130 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.004+0000 7fe8567fe700 1 -- 172.21.15.152:0/1651085770 shutdown_connections 2024-04-02T04:49:49.130 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.004+0000 7fe8567fe700 1 --2- 172.21.15.152:0/1651085770 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe850076c70 0x7fe850105440 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:49.130 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.004+0000 7fe8567fe700 1 -- 172.21.15.152:0/1651085770 >> 172.21.15.152:0/1651085770 conn(0x7fe8500fcf30 msgr2=0x7fe8500ff340 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:49.130 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.004+0000 7fe8567fe700 1 -- 172.21.15.152:0/1651085770 shutdown_connections 2024-04-02T04:49:49.130 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.004+0000 7fe8567fe700 1 -- 172.21.15.152:0/1651085770 wait complete. 2024-04-02T04:49:49.130 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.005+0000 7fe8567fe700 1 Processor -- start 2024-04-02T04:49:49.131 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.005+0000 7fe8567fe700 1 -- start start 2024-04-02T04:49:49.131 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.005+0000 7fe8567fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe850076c70 0x7fe850111730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:49.131 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.005+0000 7fe8567fe700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe850111c70 con 0x7fe850076c70 2024-04-02T04:49:49.131 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.005+0000 7fe84ffff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe850076c70 0x7fe850111730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:49.131 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.005+0000 7fe84ffff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe850076c70 0x7fe850111730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:42264/0 (socket says 172.21.15.152:42264) 2024-04-02T04:49:49.131 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.005+0000 7fe84ffff700 1 -- 172.21.15.152:0/1609771012 learned_addr learned my addr 172.21.15.152:0/1609771012 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:49.131 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.006+0000 7fe84ffff700 1 -- 172.21.15.152:0/1609771012 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe8400094f0 con 0x7fe850076c70 2024-04-02T04:49:49.131 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.006+0000 7fe84ffff700 1 --2- 172.21.15.152:0/1609771012 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe850076c70 0x7fe850111730 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7fe8400105b0 tx=0x7fe840009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:49.131 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.006+0000 7fe84cff9700 1 -- 172.21.15.152:0/1609771012 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fe840010840 con 0x7fe850076c70 2024-04-02T04:49:49.131 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.006+0000 7fe84cff9700 1 -- 172.21.15.152:0/1609771012 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fe840010e70 con 0x7fe850076c70 2024-04-02T04:49:49.131 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.006+0000 7fe84cff9700 1 -- 172.21.15.152:0/1609771012 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fe84001ae30 con 0x7fe850076c70 2024-04-02T04:49:49.131 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.006+0000 7fe8567fe700 1 -- 172.21.15.152:0/1609771012 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe850111e70 con 0x7fe850076c70 2024-04-02T04:49:49.131 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.006+0000 7fe8567fe700 1 -- 172.21.15.152:0/1609771012 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe8501129e0 con 0x7fe850076c70 2024-04-02T04:49:49.131 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.007+0000 7fe84cff9700 1 -- 172.21.15.152:0/1609771012 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 8) v1 ==== 43368+0+0 (secure 0 0 0) 0x7fe84002d3f0 con 0x7fe850076c70 2024-04-02T04:49:49.131 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.007+0000 7fe84cff9700 1 --2- 172.21.15.152:0/1609771012 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fe8380367e0 0x7fe838038c90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:49.131 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.007+0000 7fe84cff9700 1 -- 172.21.15.152:0/1609771012 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 936+0+0 (secure 0 0 0) 0x7fe84004a190 con 0x7fe850076c70 2024-04-02T04:49:49.131 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.007+0000 7fe84f7fe700 1 --2- 172.21.15.152:0/1609771012 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fe8380367e0 0x7fe838038c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:49.132 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.007+0000 7fe8567fe700 1 -- 172.21.15.152:0/1609771012 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe85005f5f0 con 0x7fe850076c70 2024-04-02T04:49:49.132 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.007+0000 7fe84f7fe700 1 --2- 172.21.15.152:0/1609771012 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fe8380367e0 0x7fe838038c90 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7fe844009990 tx=0x7fe844006e40 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:49.132 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.012+0000 7fe84cff9700 1 -- 172.21.15.152:0/1609771012 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fe84002b8e0 con 0x7fe850076c70 2024-04-02T04:49:49.132 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.214+0000 7fe8567fe700 1 -- 172.21.15.152:0/1609771012 --> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}) v1 -- 0x7fe8500fe2d0 con 0x7fe8380367e0 2024-04-02T04:49:49.132 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.218+0000 7fe84cff9700 1 -- 172.21.15.152:0/1609771012 <== mgr.14118 v2:172.21.15.152:6800/3544077480 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+28 (secure 0 0 0) 0x7fe8500fe2d0 con 0x7fe8380367e0 2024-04-02T04:49:49.132 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.220+0000 7fe8567fe700 1 -- 172.21.15.152:0/1609771012 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fe8380367e0 msgr2=0x7fe838038c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:49.132 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.220+0000 7fe8567fe700 1 --2- 172.21.15.152:0/1609771012 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fe8380367e0 0x7fe838038c90 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7fe844009990 tx=0x7fe844006e40 comp rx=0 tx=0).stop 2024-04-02T04:49:49.132 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.220+0000 7fe8567fe700 1 -- 172.21.15.152:0/1609771012 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe850076c70 msgr2=0x7fe850111730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:49.132 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.220+0000 7fe8567fe700 1 --2- 172.21.15.152:0/1609771012 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe850076c70 0x7fe850111730 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 crypto rx=0x7fe8400105b0 tx=0x7fe840009250 comp rx=0 tx=0).stop 2024-04-02T04:49:49.132 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.220+0000 7fe8567fe700 1 -- 172.21.15.152:0/1609771012 shutdown_connections 2024-04-02T04:49:49.132 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.220+0000 7fe8567fe700 1 --2- 172.21.15.152:0/1609771012 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fe8380367e0 0x7fe838038c90 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:49.132 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.220+0000 7fe8567fe700 1 --2- 172.21.15.152:0/1609771012 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe850076c70 0x7fe850111730 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:49.132 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.220+0000 7fe8567fe700 1 -- 172.21.15.152:0/1609771012 >> 172.21.15.152:0/1609771012 conn(0x7fe8500fcf30 msgr2=0x7fe8500fdb00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:49.132 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.220+0000 7fe8567fe700 1 -- 172.21.15.152:0/1609771012 shutdown_connections 2024-04-02T04:49:49.132 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:48.220+0000 7fe8567fe700 1 -- 172.21.15.152:0/1609771012 wait complete. 2024-04-02T04:49:49.132 INFO:teuthology.orchestra.run.smithi152.stdout:Deploying node-exporter service with default placement... 2024-04-02T04:49:49.399 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:49 smithi152 ceph-mon[29073]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:49:49.399 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:49 smithi152 ceph-mon[29073]: Saving service grafana spec with placement count:1 2024-04-02T04:49:49.399 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:49 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' 2024-04-02T04:49:50.252 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout Scheduled node-exporter update... 2024-04-02T04:49:50.253 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.491+0000 7f57da62c700 1 Processor -- start 2024-04-02T04:49:50.253 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.491+0000 7f57da62c700 1 -- start start 2024-04-02T04:49:50.253 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.491+0000 7f57da62c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f57d4071250 0x7f57d4105540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:50.253 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.491+0000 7f57da62c700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f57d4071660 con 0x7f57d4071250 2024-04-02T04:49:50.253 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.492+0000 7f57d3fff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f57d4071250 0x7f57d4105540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:50.253 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.492+0000 7f57d3fff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f57d4071250 0x7f57d4105540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:42266/0 (socket says 172.21.15.152:42266) 2024-04-02T04:49:50.253 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.492+0000 7f57d3fff700 1 -- 172.21.15.152:0/3243568385 learned_addr learned my addr 172.21.15.152:0/3243568385 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:50.253 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.492+0000 7f57d3fff700 1 -- 172.21.15.152:0/3243568385 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f57d4105a80 con 0x7f57d4071250 2024-04-02T04:49:50.253 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.492+0000 7f57d3fff700 1 --2- 172.21.15.152:0/3243568385 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f57d4071250 0x7f57d4105540 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f57c40098e0 tx=0x7f57c4009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=9c2f9377fe397470 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:50.253 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.492+0000 7f57d2ffd700 1 -- 172.21.15.152:0/3243568385 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f57c400b860 con 0x7f57d4071250 2024-04-02T04:49:50.253 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.492+0000 7f57d2ffd700 1 -- 172.21.15.152:0/3243568385 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f57c400be90 con 0x7f57d4071250 2024-04-02T04:49:50.253 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.492+0000 7f57d2ffd700 1 -- 172.21.15.152:0/3243568385 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f57c40103c0 con 0x7f57d4071250 2024-04-02T04:49:50.253 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.493+0000 7f57da62c700 1 -- 172.21.15.152:0/3243568385 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f57d4071250 msgr2=0x7f57d4105540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:50.253 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.493+0000 7f57da62c700 1 --2- 172.21.15.152:0/3243568385 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f57d4071250 0x7f57d4105540 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f57c40098e0 tx=0x7f57c4009250 comp rx=0 tx=0).stop 2024-04-02T04:49:50.253 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.493+0000 7f57da62c700 1 -- 172.21.15.152:0/3243568385 shutdown_connections 2024-04-02T04:49:50.254 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.493+0000 7f57da62c700 1 --2- 172.21.15.152:0/3243568385 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f57d4071250 0x7f57d4105540 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:50.254 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.493+0000 7f57da62c700 1 -- 172.21.15.152:0/3243568385 >> 172.21.15.152:0/3243568385 conn(0x7f57d40fcfb0 msgr2=0x7f57d40ff3c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:50.254 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.493+0000 7f57da62c700 1 -- 172.21.15.152:0/3243568385 shutdown_connections 2024-04-02T04:49:50.254 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.493+0000 7f57da62c700 1 -- 172.21.15.152:0/3243568385 wait complete. 2024-04-02T04:49:50.254 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.493+0000 7f57da62c700 1 Processor -- start 2024-04-02T04:49:50.254 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.494+0000 7f57da62c700 1 -- start start 2024-04-02T04:49:50.254 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.494+0000 7f57da62c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f57d4071250 0x7f57d4111760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:50.254 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.494+0000 7f57da62c700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f57d4111ca0 con 0x7f57d4071250 2024-04-02T04:49:50.254 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.494+0000 7f57d3fff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f57d4071250 0x7f57d4111760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:50.254 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.494+0000 7f57d3fff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f57d4071250 0x7f57d4111760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:42274/0 (socket says 172.21.15.152:42274) 2024-04-02T04:49:50.254 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.494+0000 7f57d3fff700 1 -- 172.21.15.152:0/2739048283 learned_addr learned my addr 172.21.15.152:0/2739048283 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:50.254 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.494+0000 7f57d3fff700 1 -- 172.21.15.152:0/2739048283 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f57c40094f0 con 0x7f57d4071250 2024-04-02T04:49:50.254 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.494+0000 7f57d3fff700 1 --2- 172.21.15.152:0/2739048283 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f57d4071250 0x7f57d4111760 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f57c40105b0 tx=0x7f57c4009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:50.254 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.494+0000 7f57d0ff9700 1 -- 172.21.15.152:0/2739048283 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f57c4010840 con 0x7f57d4071250 2024-04-02T04:49:50.254 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.494+0000 7f57da62c700 1 -- 172.21.15.152:0/2739048283 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f57d4111ea0 con 0x7f57d4071250 2024-04-02T04:49:50.254 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.494+0000 7f57d0ff9700 1 -- 172.21.15.152:0/2739048283 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f57c4010e70 con 0x7f57d4071250 2024-04-02T04:49:50.255 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.494+0000 7f57d0ff9700 1 -- 172.21.15.152:0/2739048283 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f57c401ae30 con 0x7f57d4071250 2024-04-02T04:49:50.255 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.495+0000 7f57da62c700 1 -- 172.21.15.152:0/2739048283 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f57d4112a10 con 0x7f57d4071250 2024-04-02T04:49:50.255 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.495+0000 7f57d0ff9700 1 -- 172.21.15.152:0/2739048283 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 8) v1 ==== 43368+0+0 (secure 0 0 0) 0x7f57c402d3f0 con 0x7f57d4071250 2024-04-02T04:49:50.255 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.495+0000 7f57da62c700 1 -- 172.21.15.152:0/2739048283 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f57d405f5f0 con 0x7f57d4071250 2024-04-02T04:49:50.255 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.496+0000 7f57d0ff9700 1 --2- 172.21.15.152:0/2739048283 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f57bc036790 0x7f57bc038c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:50.255 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.496+0000 7f57d0ff9700 1 -- 172.21.15.152:0/2739048283 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 936+0+0 (secure 0 0 0) 0x7f57c404a280 con 0x7f57d4071250 2024-04-02T04:49:50.255 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.496+0000 7f57d37fe700 1 --2- 172.21.15.152:0/2739048283 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f57bc036790 0x7f57bc038c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:50.255 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.496+0000 7f57d37fe700 1 --2- 172.21.15.152:0/2739048283 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f57bc036790 0x7f57bc038c40 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f57c8009990 tx=0x7f57c8006e40 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:50.255 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.500+0000 7f57d0ff9700 1 -- 172.21.15.152:0/2739048283 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f57c402b8e0 con 0x7f57d4071250 2024-04-02T04:49:50.255 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.694+0000 7f57da62c700 1 -- 172.21.15.152:0/2739048283 --> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}) v1 -- 0x7f57d40fe3a0 con 0x7f57bc036790 2024-04-02T04:49:50.255 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.699+0000 7f57d0ff9700 1 -- 172.21.15.152:0/2739048283 <== mgr.14118 v2:172.21.15.152:6800/3544077480 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+34 (secure 0 0 0) 0x7f57d40fe3a0 con 0x7f57bc036790 2024-04-02T04:49:50.255 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.701+0000 7f57da62c700 1 -- 172.21.15.152:0/2739048283 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f57bc036790 msgr2=0x7f57bc038c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:50.255 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.701+0000 7f57da62c700 1 --2- 172.21.15.152:0/2739048283 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f57bc036790 0x7f57bc038c40 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7f57c8009990 tx=0x7f57c8006e40 comp rx=0 tx=0).stop 2024-04-02T04:49:50.255 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.701+0000 7f57da62c700 1 -- 172.21.15.152:0/2739048283 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f57d4071250 msgr2=0x7f57d4111760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:50.255 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.701+0000 7f57da62c700 1 --2- 172.21.15.152:0/2739048283 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f57d4071250 0x7f57d4111760 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 crypto rx=0x7f57c40105b0 tx=0x7f57c4009250 comp rx=0 tx=0).stop 2024-04-02T04:49:50.255 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.701+0000 7f57da62c700 1 -- 172.21.15.152:0/2739048283 shutdown_connections 2024-04-02T04:49:50.256 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.701+0000 7f57da62c700 1 --2- 172.21.15.152:0/2739048283 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f57bc036790 0x7f57bc038c40 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:50.256 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.701+0000 7f57da62c700 1 --2- 172.21.15.152:0/2739048283 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f57d4071250 0x7f57d4111760 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:50.256 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.701+0000 7f57da62c700 1 -- 172.21.15.152:0/2739048283 >> 172.21.15.152:0/2739048283 conn(0x7f57d40fcfb0 msgr2=0x7f57d40fdbd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:50.256 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.701+0000 7f57da62c700 1 -- 172.21.15.152:0/2739048283 shutdown_connections 2024-04-02T04:49:50.256 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:49.701+0000 7f57da62c700 1 -- 172.21.15.152:0/2739048283 wait complete. 2024-04-02T04:49:50.256 INFO:teuthology.orchestra.run.smithi152.stdout:Deploying alertmanager service with default placement... 2024-04-02T04:49:51.006 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:50 smithi152 ceph-mon[29073]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:49:51.006 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:50 smithi152 ceph-mon[29073]: Saving service node-exporter spec with placement * 2024-04-02T04:49:51.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:50 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' 2024-04-02T04:49:51.476 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout Scheduled alertmanager update... 2024-04-02T04:49:51.476 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.658+0000 7fe8b692e700 1 Processor -- start 2024-04-02T04:49:51.477 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.658+0000 7fe8b692e700 1 -- start start 2024-04-02T04:49:51.477 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.658+0000 7fe8b692e700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe8b0102920 0x7fe8b0102d30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:51.477 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.658+0000 7fe8b692e700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe8b0103270 con 0x7fe8b0102920 2024-04-02T04:49:51.477 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.658+0000 7fe8affff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe8b0102920 0x7fe8b0102d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:51.477 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.659+0000 7fe8affff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe8b0102920 0x7fe8b0102d30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:44090/0 (socket says 172.21.15.152:44090) 2024-04-02T04:49:51.477 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.659+0000 7fe8affff700 1 -- 172.21.15.152:0/3481476389 learned_addr learned my addr 172.21.15.152:0/3481476389 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:51.477 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.659+0000 7fe8affff700 1 -- 172.21.15.152:0/3481476389 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe8b01033b0 con 0x7fe8b0102920 2024-04-02T04:49:51.477 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.659+0000 7fe8affff700 1 --2- 172.21.15.152:0/3481476389 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe8b0102920 0x7fe8b0102d30 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7fe8a00098e0 tx=0x7fe8a0009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=95c7d2788c78e583 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:51.477 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.659+0000 7fe8aeffd700 1 -- 172.21.15.152:0/3481476389 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fe8a000b860 con 0x7fe8b0102920 2024-04-02T04:49:51.477 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.659+0000 7fe8aeffd700 1 -- 172.21.15.152:0/3481476389 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fe8a000be90 con 0x7fe8b0102920 2024-04-02T04:49:51.477 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.659+0000 7fe8aeffd700 1 -- 172.21.15.152:0/3481476389 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fe8a00103c0 con 0x7fe8b0102920 2024-04-02T04:49:51.477 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.660+0000 7fe8b692e700 1 -- 172.21.15.152:0/3481476389 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe8b0102920 msgr2=0x7fe8b0102d30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:51.477 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.660+0000 7fe8b692e700 1 --2- 172.21.15.152:0/3481476389 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe8b0102920 0x7fe8b0102d30 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 crypto rx=0x7fe8a00098e0 tx=0x7fe8a0009250 comp rx=0 tx=0).stop 2024-04-02T04:49:51.477 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.660+0000 7fe8b692e700 1 -- 172.21.15.152:0/3481476389 shutdown_connections 2024-04-02T04:49:51.478 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.660+0000 7fe8b692e700 1 --2- 172.21.15.152:0/3481476389 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe8b0102920 0x7fe8b0102d30 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:51.478 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.660+0000 7fe8b692e700 1 -- 172.21.15.152:0/3481476389 >> 172.21.15.152:0/3481476389 conn(0x7fe8b00fe0a0 msgr2=0x7fe8b01004b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:51.478 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.660+0000 7fe8b692e700 1 -- 172.21.15.152:0/3481476389 shutdown_connections 2024-04-02T04:49:51.478 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.660+0000 7fe8b692e700 1 -- 172.21.15.152:0/3481476389 wait complete. 2024-04-02T04:49:51.478 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.660+0000 7fe8b692e700 1 Processor -- start 2024-04-02T04:49:51.478 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.660+0000 7fe8b692e700 1 -- start start 2024-04-02T04:49:51.478 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.661+0000 7fe8b692e700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe8b0102920 0x7fe8b01116e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:51.478 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.661+0000 7fe8b692e700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe8b0111c20 con 0x7fe8b0102920 2024-04-02T04:49:51.478 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.661+0000 7fe8affff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe8b0102920 0x7fe8b01116e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:51.478 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.661+0000 7fe8affff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe8b0102920 0x7fe8b01116e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:44094/0 (socket says 172.21.15.152:44094) 2024-04-02T04:49:51.478 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.661+0000 7fe8affff700 1 -- 172.21.15.152:0/768648546 learned_addr learned my addr 172.21.15.152:0/768648546 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:51.478 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.661+0000 7fe8affff700 1 -- 172.21.15.152:0/768648546 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe8a00094f0 con 0x7fe8b0102920 2024-04-02T04:49:51.478 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.661+0000 7fe8affff700 1 --2- 172.21.15.152:0/768648546 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe8b0102920 0x7fe8b01116e0 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7fe8a00090b0 tx=0x7fe8a0009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:51.479 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.661+0000 7fe8acff9700 1 -- 172.21.15.152:0/768648546 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fe8a0010840 con 0x7fe8b0102920 2024-04-02T04:49:51.479 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.661+0000 7fe8acff9700 1 -- 172.21.15.152:0/768648546 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fe8a0010e70 con 0x7fe8b0102920 2024-04-02T04:49:51.479 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.661+0000 7fe8b692e700 1 -- 172.21.15.152:0/768648546 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe8b0111e20 con 0x7fe8b0102920 2024-04-02T04:49:51.479 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.661+0000 7fe8acff9700 1 -- 172.21.15.152:0/768648546 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fe8a0024e40 con 0x7fe8b0102920 2024-04-02T04:49:51.479 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.661+0000 7fe8b692e700 1 -- 172.21.15.152:0/768648546 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe8b0112990 con 0x7fe8b0102920 2024-04-02T04:49:51.479 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.662+0000 7fe8acff9700 1 -- 172.21.15.152:0/768648546 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 8) v1 ==== 43368+0+0 (secure 0 0 0) 0x7fe8a001a510 con 0x7fe8b0102920 2024-04-02T04:49:51.479 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.662+0000 7fe8acff9700 1 --2- 172.21.15.152:0/768648546 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fe898036740 0x7fe898038bf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:51.479 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.662+0000 7fe8acff9700 1 -- 172.21.15.152:0/768648546 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 936+0+0 (secure 0 0 0) 0x7fe8a004a830 con 0x7fe8b0102920 2024-04-02T04:49:51.479 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.662+0000 7fe8af7fe700 1 --2- 172.21.15.152:0/768648546 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fe898036740 0x7fe898038bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:51.479 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.662+0000 7fe8b692e700 1 -- 172.21.15.152:0/768648546 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe8b005f5f0 con 0x7fe8b0102920 2024-04-02T04:49:51.479 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.663+0000 7fe8af7fe700 1 --2- 172.21.15.152:0/768648546 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fe898036740 0x7fe898038bf0 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7fe8a4009990 tx=0x7fe8a4006e40 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:51.479 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.667+0000 7fe8acff9700 1 -- 172.21.15.152:0/768648546 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fe8a0018940 con 0x7fe8b0102920 2024-04-02T04:49:51.479 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.866+0000 7fe8b692e700 1 -- 172.21.15.152:0/768648546 --> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}) v1 -- 0x7fe8b00ff330 con 0x7fe898036740 2024-04-02T04:49:51.479 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.873+0000 7fe8acff9700 1 -- 172.21.15.152:0/768648546 <== mgr.14118 v2:172.21.15.152:6800/3544077480 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+33 (secure 0 0 0) 0x7fe8b00ff330 con 0x7fe898036740 2024-04-02T04:49:51.480 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.875+0000 7fe8b692e700 1 -- 172.21.15.152:0/768648546 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fe898036740 msgr2=0x7fe898038bf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:51.480 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.875+0000 7fe8b692e700 1 --2- 172.21.15.152:0/768648546 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fe898036740 0x7fe898038bf0 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7fe8a4009990 tx=0x7fe8a4006e40 comp rx=0 tx=0).stop 2024-04-02T04:49:51.480 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.875+0000 7fe8b692e700 1 -- 172.21.15.152:0/768648546 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe8b0102920 msgr2=0x7fe8b01116e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:51.480 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.875+0000 7fe8b692e700 1 --2- 172.21.15.152:0/768648546 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe8b0102920 0x7fe8b01116e0 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7fe8a00090b0 tx=0x7fe8a0009250 comp rx=0 tx=0).stop 2024-04-02T04:49:51.480 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.875+0000 7fe8b692e700 1 -- 172.21.15.152:0/768648546 shutdown_connections 2024-04-02T04:49:51.480 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.875+0000 7fe8b692e700 1 --2- 172.21.15.152:0/768648546 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fe898036740 0x7fe898038bf0 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:51.480 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.875+0000 7fe8b692e700 1 --2- 172.21.15.152:0/768648546 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe8b0102920 0x7fe8b01116e0 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:51.480 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.875+0000 7fe8b692e700 1 -- 172.21.15.152:0/768648546 >> 172.21.15.152:0/768648546 conn(0x7fe8b00fe0a0 msgr2=0x7fe8b00fec20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:51.480 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.875+0000 7fe8b692e700 1 -- 172.21.15.152:0/768648546 shutdown_connections 2024-04-02T04:49:51.480 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:50.875+0000 7fe8b692e700 1 -- 172.21.15.152:0/768648546 wait complete. 2024-04-02T04:49:52.069 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:51 smithi152 ceph-mon[29073]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:49:52.069 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:51 smithi152 ceph-mon[29073]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:49:52.069 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:51 smithi152 ceph-mon[29073]: Saving service alertmanager spec with placement count:1 2024-04-02T04:49:52.070 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:51 smithi152 ceph-mon[29073]: from='mgr.14118 172.21.15.152:0/741958570' entity='mgr.smithi152.jivdcl' 2024-04-02T04:49:52.649 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.869+0000 7fccf50fd700 1 Processor -- start 2024-04-02T04:49:52.649 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.869+0000 7fccf50fd700 1 -- start start 2024-04-02T04:49:52.649 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.869+0000 7fccf50fd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fccf0076b60 0x7fccf0105540 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:52.649 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.869+0000 7fccf50fd700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fccf0076f70 con 0x7fccf0076b60 2024-04-02T04:49:52.650 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.869+0000 7fcceed9d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fccf0076b60 0x7fccf0105540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:52.650 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.869+0000 7fcceed9d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fccf0076b60 0x7fccf0105540 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:44110/0 (socket says 172.21.15.152:44110) 2024-04-02T04:49:52.650 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.869+0000 7fcceed9d700 1 -- 172.21.15.152:0/2418032145 learned_addr learned my addr 172.21.15.152:0/2418032145 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:52.650 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.869+0000 7fcceed9d700 1 -- 172.21.15.152:0/2418032145 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fccf0105a80 con 0x7fccf0076b60 2024-04-02T04:49:52.650 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.869+0000 7fcceed9d700 1 --2- 172.21.15.152:0/2418032145 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fccf0076b60 0x7fccf0105540 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7fcce00098e0 tx=0x7fcce0009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=ee62b59ceb7b9d0b server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:52.650 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.870+0000 7fccedd9b700 1 -- 172.21.15.152:0/2418032145 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fcce000b860 con 0x7fccf0076b60 2024-04-02T04:49:52.650 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.870+0000 7fccedd9b700 1 -- 172.21.15.152:0/2418032145 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fcce000be90 con 0x7fccf0076b60 2024-04-02T04:49:52.650 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.870+0000 7fccedd9b700 1 -- 172.21.15.152:0/2418032145 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fcce00103c0 con 0x7fccf0076b60 2024-04-02T04:49:52.650 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.870+0000 7fccf50fd700 1 -- 172.21.15.152:0/2418032145 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fccf0076b60 msgr2=0x7fccf0105540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:52.650 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.870+0000 7fccf50fd700 1 --2- 172.21.15.152:0/2418032145 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fccf0076b60 0x7fccf0105540 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 crypto rx=0x7fcce00098e0 tx=0x7fcce0009250 comp rx=0 tx=0).stop 2024-04-02T04:49:52.650 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.870+0000 7fccf50fd700 1 -- 172.21.15.152:0/2418032145 shutdown_connections 2024-04-02T04:49:52.650 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.870+0000 7fccf50fd700 1 --2- 172.21.15.152:0/2418032145 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fccf0076b60 0x7fccf0105540 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:52.650 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.870+0000 7fccf50fd700 1 -- 172.21.15.152:0/2418032145 >> 172.21.15.152:0/2418032145 conn(0x7fccf00fcf90 msgr2=0x7fccf00ff3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:52.650 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.870+0000 7fccf50fd700 1 -- 172.21.15.152:0/2418032145 shutdown_connections 2024-04-02T04:49:52.650 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.870+0000 7fccf50fd700 1 -- 172.21.15.152:0/2418032145 wait complete. 2024-04-02T04:49:52.651 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.871+0000 7fccf50fd700 1 Processor -- start 2024-04-02T04:49:52.651 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.871+0000 7fccf50fd700 1 -- start start 2024-04-02T04:49:52.651 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.871+0000 7fccf50fd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fccf0076b60 0x7fccf0102e90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:52.651 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.871+0000 7fccf50fd700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fccf01033d0 con 0x7fccf0076b60 2024-04-02T04:49:52.651 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.871+0000 7fcceed9d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fccf0076b60 0x7fccf0102e90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:52.651 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.871+0000 7fcceed9d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fccf0076b60 0x7fccf0102e90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:44112/0 (socket says 172.21.15.152:44112) 2024-04-02T04:49:52.651 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.871+0000 7fcceed9d700 1 -- 172.21.15.152:0/1575944484 learned_addr learned my addr 172.21.15.152:0/1575944484 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:52.651 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.871+0000 7fcceed9d700 1 -- 172.21.15.152:0/1575944484 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcce00094f0 con 0x7fccf0076b60 2024-04-02T04:49:52.651 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.872+0000 7fcceed9d700 1 --2- 172.21.15.152:0/1575944484 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fccf0076b60 0x7fccf0102e90 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7fcce0006d40 tx=0x7fcce0009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:52.651 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.875+0000 7fccd77fe700 1 -- 172.21.15.152:0/1575944484 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fcce0010840 con 0x7fccf0076b60 2024-04-02T04:49:52.651 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.875+0000 7fccd77fe700 1 -- 172.21.15.152:0/1575944484 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fcce0010e70 con 0x7fccf0076b60 2024-04-02T04:49:52.651 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.876+0000 7fccf50fd700 1 -- 172.21.15.152:0/1575944484 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fccf0101440 con 0x7fccf0076b60 2024-04-02T04:49:52.651 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.876+0000 7fccd77fe700 1 -- 172.21.15.152:0/1575944484 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fcce0024ea0 con 0x7fccf0076b60 2024-04-02T04:49:52.651 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.876+0000 7fccf50fd700 1 -- 172.21.15.152:0/1575944484 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fccf0101930 con 0x7fccf0076b60 2024-04-02T04:49:52.651 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.876+0000 7fccf50fd700 1 -- 172.21.15.152:0/1575944484 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fccf004bed0 con 0x7fccf0076b60 2024-04-02T04:49:52.652 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.877+0000 7fccd77fe700 1 -- 172.21.15.152:0/1575944484 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 8) v1 ==== 43368+0+0 (secure 0 0 0) 0x7fcce001a510 con 0x7fccf0076b60 2024-04-02T04:49:52.652 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.877+0000 7fccd77fe700 1 --2- 172.21.15.152:0/1575944484 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fccd8036740 0x7fccd8038bf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:52.652 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.877+0000 7fccd77fe700 1 -- 172.21.15.152:0/1575944484 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 936+0+0 (secure 0 0 0) 0x7fcce004a870 con 0x7fccf0076b60 2024-04-02T04:49:52.652 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.877+0000 7fccee59c700 1 --2- 172.21.15.152:0/1575944484 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fccd8036740 0x7fccd8038bf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:52.652 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.877+0000 7fccee59c700 1 --2- 172.21.15.152:0/1575944484 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fccd8036740 0x7fccd8038bf0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7fcce4009990 tx=0x7fcce4006e40 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:52.652 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:51.881+0000 7fccd77fe700 1 -- 172.21.15.152:0/1575944484 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fcce0018530 con 0x7fccf0076b60 2024-04-02T04:49:52.652 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:52.066+0000 7fccf50fd700 1 -- 172.21.15.152:0/1575944484 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command([{prefix=config set, name=mgr/cephadm/container_init}] v 0) v1 -- 0x7fccf01068c0 con 0x7fccf0076b60 2024-04-02T04:49:52.652 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:52.071+0000 7fccd77fe700 1 -- 172.21.15.152:0/1575944484 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/cephadm/container_init}]=0 v7)=0 v7) v1 ==== 142+0+0 (secure 0 0 0) 0x7fcce002d3a0 con 0x7fccf0076b60 2024-04-02T04:49:52.652 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:52.073+0000 7fccf50fd700 1 -- 172.21.15.152:0/1575944484 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fccd8036740 msgr2=0x7fccd8038bf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:52.652 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:52.073+0000 7fccf50fd700 1 --2- 172.21.15.152:0/1575944484 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fccd8036740 0x7fccd8038bf0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7fcce4009990 tx=0x7fcce4006e40 comp rx=0 tx=0).stop 2024-04-02T04:49:52.652 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:52.073+0000 7fccf50fd700 1 -- 172.21.15.152:0/1575944484 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fccf0076b60 msgr2=0x7fccf0102e90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:52.652 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:52.073+0000 7fccf50fd700 1 --2- 172.21.15.152:0/1575944484 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fccf0076b60 0x7fccf0102e90 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 crypto rx=0x7fcce0006d40 tx=0x7fcce0009250 comp rx=0 tx=0).stop 2024-04-02T04:49:52.652 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:52.074+0000 7fccf50fd700 1 -- 172.21.15.152:0/1575944484 shutdown_connections 2024-04-02T04:49:52.652 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:52.074+0000 7fccf50fd700 1 --2- 172.21.15.152:0/1575944484 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7fccd8036740 0x7fccd8038bf0 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:52.652 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:52.074+0000 7fccf50fd700 1 --2- 172.21.15.152:0/1575944484 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fccf0076b60 0x7fccf0102e90 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:52.653 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:52.074+0000 7fccf50fd700 1 -- 172.21.15.152:0/1575944484 >> 172.21.15.152:0/1575944484 conn(0x7fccf00fcf90 msgr2=0x7fccf00fdba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:52.653 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:52.074+0000 7fccf50fd700 1 -- 172.21.15.152:0/1575944484 shutdown_connections 2024-04-02T04:49:52.653 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:52.074+0000 7fccf50fd700 1 -- 172.21.15.152:0/1575944484 wait complete. 2024-04-02T04:49:53.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:53 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/1575944484' entity='client.admin' 2024-04-02T04:49:53.815 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.077+0000 7f759befe700 1 Processor -- start 2024-04-02T04:49:53.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.078+0000 7f759befe700 1 -- start start 2024-04-02T04:49:53.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.078+0000 7f759befe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7594076c50 0x7f7594105460 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:53.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.078+0000 7f759befe700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f75941059a0 con 0x7f7594076c50 2024-04-02T04:49:53.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.078+0000 7f7599c9a700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7594076c50 0x7f7594105460 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:53.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.078+0000 7f7599c9a700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7594076c50 0x7f7594105460 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:44128/0 (socket says 172.21.15.152:44128) 2024-04-02T04:49:53.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.078+0000 7f7599c9a700 1 -- 172.21.15.152:0/2145528893 learned_addr learned my addr 172.21.15.152:0/2145528893 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:53.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.078+0000 7f7599c9a700 1 -- 172.21.15.152:0/2145528893 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7594105ae0 con 0x7f7594076c50 2024-04-02T04:49:53.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.078+0000 7f7599c9a700 1 --2- 172.21.15.152:0/2145528893 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7594076c50 0x7f7594105460 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f75900098e0 tx=0x7f7590009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=3fd14b440c049410 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:53.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.079+0000 7f7598c98700 1 -- 172.21.15.152:0/2145528893 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f759000b860 con 0x7f7594076c50 2024-04-02T04:49:53.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.079+0000 7f7598c98700 1 -- 172.21.15.152:0/2145528893 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f759000be90 con 0x7f7594076c50 2024-04-02T04:49:53.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.079+0000 7f7598c98700 1 -- 172.21.15.152:0/2145528893 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f75900103c0 con 0x7f7594076c50 2024-04-02T04:49:53.817 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.080+0000 7f759befe700 1 -- 172.21.15.152:0/2145528893 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7594076c50 msgr2=0x7f7594105460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:53.817 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.080+0000 7f759befe700 1 --2- 172.21.15.152:0/2145528893 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7594076c50 0x7f7594105460 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 crypto rx=0x7f75900098e0 tx=0x7f7590009250 comp rx=0 tx=0).stop 2024-04-02T04:49:53.817 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.080+0000 7f759befe700 1 -- 172.21.15.152:0/2145528893 shutdown_connections 2024-04-02T04:49:53.817 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.080+0000 7f759befe700 1 --2- 172.21.15.152:0/2145528893 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7594076c50 0x7f7594105460 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:53.817 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.080+0000 7f759befe700 1 -- 172.21.15.152:0/2145528893 >> 172.21.15.152:0/2145528893 conn(0x7f75940fcf30 msgr2=0x7f75940ff340 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:53.817 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.080+0000 7f759befe700 1 -- 172.21.15.152:0/2145528893 shutdown_connections 2024-04-02T04:49:53.817 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.080+0000 7f759befe700 1 -- 172.21.15.152:0/2145528893 wait complete. 2024-04-02T04:49:53.817 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.080+0000 7f759befe700 1 Processor -- start 2024-04-02T04:49:53.817 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.081+0000 7f759befe700 1 -- start start 2024-04-02T04:49:53.817 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.081+0000 7f759befe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7594076c50 0x7f7594111760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:53.817 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.081+0000 7f759befe700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7594111ca0 con 0x7f7594076c50 2024-04-02T04:49:53.817 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.081+0000 7f7599c9a700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7594076c50 0x7f7594111760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:53.818 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.081+0000 7f7599c9a700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7594076c50 0x7f7594111760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:44140/0 (socket says 172.21.15.152:44140) 2024-04-02T04:49:53.818 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.081+0000 7f7599c9a700 1 -- 172.21.15.152:0/2592446572 learned_addr learned my addr 172.21.15.152:0/2592446572 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:53.818 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.081+0000 7f7599c9a700 1 -- 172.21.15.152:0/2592446572 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f75900094f0 con 0x7f7594076c50 2024-04-02T04:49:53.818 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.081+0000 7f7599c9a700 1 --2- 172.21.15.152:0/2592446572 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7594076c50 0x7f7594111760 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f75900105b0 tx=0x7f7590009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:53.818 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.081+0000 7f758a7fc700 1 -- 172.21.15.152:0/2592446572 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7590010840 con 0x7f7594076c50 2024-04-02T04:49:53.818 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.081+0000 7f758a7fc700 1 -- 172.21.15.152:0/2592446572 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f7590010e70 con 0x7f7594076c50 2024-04-02T04:49:53.818 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.081+0000 7f758a7fc700 1 -- 172.21.15.152:0/2592446572 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f759001ae30 con 0x7f7594076c50 2024-04-02T04:49:53.818 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.081+0000 7f759befe700 1 -- 172.21.15.152:0/2592446572 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7594111ea0 con 0x7f7594076c50 2024-04-02T04:49:53.818 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.081+0000 7f759befe700 1 -- 172.21.15.152:0/2592446572 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7594112a10 con 0x7f7594076c50 2024-04-02T04:49:53.818 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.082+0000 7f758a7fc700 1 -- 172.21.15.152:0/2592446572 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 8) v1 ==== 43368+0+0 (secure 0 0 0) 0x7f759002d3f0 con 0x7f7594076c50 2024-04-02T04:49:53.818 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.083+0000 7f758a7fc700 1 --2- 172.21.15.152:0/2592446572 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f75800367e0 0x7f7580038c90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:53.819 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.083+0000 7f759befe700 1 -- 172.21.15.152:0/2592446572 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f759405f5f0 con 0x7f7594076c50 2024-04-02T04:49:53.819 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.083+0000 7f7599499700 1 --2- 172.21.15.152:0/2592446572 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f75800367e0 0x7f7580038c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:53.819 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.083+0000 7f758a7fc700 1 -- 172.21.15.152:0/2592446572 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 936+0+0 (secure 0 0 0) 0x7f759004a140 con 0x7f7594076c50 2024-04-02T04:49:53.819 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.083+0000 7f7599499700 1 --2- 172.21.15.152:0/2592446572 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f75800367e0 0x7f7580038c90 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f7584009990 tx=0x7f7584006e40 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:53.819 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.087+0000 7f758a7fc700 1 -- 172.21.15.152:0/2592446572 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f75900109a0 con 0x7f7594076c50 2024-04-02T04:49:53.819 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.274+0000 7f759befe700 1 -- 172.21.15.152:0/2592446572 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command([{prefix=config set, name=mgr/dashboard/ssl_server_port}] v 0) v1 -- 0x7f759404bed0 con 0x7f7594076c50 2024-04-02T04:49:53.819 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.280+0000 7f758a7fc700 1 -- 172.21.15.152:0/2592446572 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/dashboard/ssl_server_port}]=0 v8)=0 v8) v1 ==== 130+0+0 (secure 0 0 0) 0x7f759002b2f0 con 0x7f7594076c50 2024-04-02T04:49:53.819 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.282+0000 7f759befe700 1 -- 172.21.15.152:0/2592446572 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f75800367e0 msgr2=0x7f7580038c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:53.819 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.282+0000 7f759befe700 1 --2- 172.21.15.152:0/2592446572 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f75800367e0 0x7f7580038c90 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f7584009990 tx=0x7f7584006e40 comp rx=0 tx=0).stop 2024-04-02T04:49:53.819 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.282+0000 7f759befe700 1 -- 172.21.15.152:0/2592446572 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7594076c50 msgr2=0x7f7594111760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:53.819 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.282+0000 7f759befe700 1 --2- 172.21.15.152:0/2592446572 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7594076c50 0x7f7594111760 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7f75900105b0 tx=0x7f7590009250 comp rx=0 tx=0).stop 2024-04-02T04:49:53.820 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.282+0000 7f759befe700 1 -- 172.21.15.152:0/2592446572 shutdown_connections 2024-04-02T04:49:53.820 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.282+0000 7f759befe700 1 --2- 172.21.15.152:0/2592446572 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f75800367e0 0x7f7580038c90 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:53.820 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.282+0000 7f759befe700 1 --2- 172.21.15.152:0/2592446572 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7594076c50 0x7f7594111760 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:53.820 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.282+0000 7f759befe700 1 -- 172.21.15.152:0/2592446572 >> 172.21.15.152:0/2592446572 conn(0x7f75940fcf30 msgr2=0x7f75940fdb10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:53.820 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.283+0000 7f759befe700 1 -- 172.21.15.152:0/2592446572 shutdown_connections 2024-04-02T04:49:53.820 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:53.283+0000 7f759befe700 1 -- 172.21.15.152:0/2592446572 wait complete. 2024-04-02T04:49:53.820 INFO:teuthology.orchestra.run.smithi152.stdout:Enabling the dashboard module... 2024-04-02T04:49:54.467 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:54 smithi152 ceph-mon[29073]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:49:54.468 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:54 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/2592446572' entity='client.admin' 2024-04-02T04:49:55.599 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:55 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/742064498' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-02T04:49:55.875 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.216+0000 7f7b6b0d7700 1 Processor -- start 2024-04-02T04:49:55.875 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.216+0000 7f7b6b0d7700 1 -- start start 2024-04-02T04:49:55.875 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.217+0000 7f7b6b0d7700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7b641028f0 0x7f7b64102d00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:55.875 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.217+0000 7f7b6b0d7700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7b64103240 con 0x7f7b641028f0 2024-04-02T04:49:55.875 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.217+0000 7f7b68e73700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7b641028f0 0x7f7b64102d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:55.875 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.217+0000 7f7b68e73700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7b641028f0 0x7f7b64102d00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:44144/0 (socket says 172.21.15.152:44144) 2024-04-02T04:49:55.875 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.217+0000 7f7b68e73700 1 -- 172.21.15.152:0/1382337697 learned_addr learned my addr 172.21.15.152:0/1382337697 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:55.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.217+0000 7f7b68e73700 1 -- 172.21.15.152:0/1382337697 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7b64103380 con 0x7f7b641028f0 2024-04-02T04:49:55.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.217+0000 7f7b68e73700 1 --2- 172.21.15.152:0/1382337697 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7b641028f0 0x7f7b64102d00 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f7b5c009e50 tx=0x7f7b5c009840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=1a36be0997153ba server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:55.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.217+0000 7f7b637fe700 1 -- 172.21.15.152:0/1382337697 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7b5c0092e0 con 0x7f7b641028f0 2024-04-02T04:49:55.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.217+0000 7f7b637fe700 1 -- 172.21.15.152:0/1382337697 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f7b5c012430 con 0x7f7b641028f0 2024-04-02T04:49:55.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.217+0000 7f7b637fe700 1 -- 172.21.15.152:0/1382337697 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7b5c0075a0 con 0x7f7b641028f0 2024-04-02T04:49:55.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.218+0000 7f7b6b0d7700 1 -- 172.21.15.152:0/1382337697 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7b641028f0 msgr2=0x7f7b64102d00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:55.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.218+0000 7f7b6b0d7700 1 --2- 172.21.15.152:0/1382337697 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7b641028f0 0x7f7b64102d00 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f7b5c009e50 tx=0x7f7b5c009840 comp rx=0 tx=0).stop 2024-04-02T04:49:55.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.218+0000 7f7b6b0d7700 1 -- 172.21.15.152:0/1382337697 shutdown_connections 2024-04-02T04:49:55.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.218+0000 7f7b6b0d7700 1 --2- 172.21.15.152:0/1382337697 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7b641028f0 0x7f7b64102d00 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:55.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.218+0000 7f7b6b0d7700 1 -- 172.21.15.152:0/1382337697 >> 172.21.15.152:0/1382337697 conn(0x7f7b640fe0d0 msgr2=0x7f7b641004e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:55.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.218+0000 7f7b6b0d7700 1 -- 172.21.15.152:0/1382337697 shutdown_connections 2024-04-02T04:49:55.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.218+0000 7f7b6b0d7700 1 -- 172.21.15.152:0/1382337697 wait complete. 2024-04-02T04:49:55.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.218+0000 7f7b6b0d7700 1 Processor -- start 2024-04-02T04:49:55.876 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.218+0000 7f7b6b0d7700 1 -- start start 2024-04-02T04:49:55.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.219+0000 7f7b6b0d7700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7b641028f0 0x7f7b64111700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:55.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.219+0000 7f7b6b0d7700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7b64111c40 con 0x7f7b641028f0 2024-04-02T04:49:55.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.219+0000 7f7b68e73700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7b641028f0 0x7f7b64111700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:55.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.219+0000 7f7b68e73700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7b641028f0 0x7f7b64111700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:44150/0 (socket says 172.21.15.152:44150) 2024-04-02T04:49:55.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.219+0000 7f7b68e73700 1 -- 172.21.15.152:0/742064498 learned_addr learned my addr 172.21.15.152:0/742064498 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:55.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.219+0000 7f7b68e73700 1 -- 172.21.15.152:0/742064498 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7b5c009ae0 con 0x7f7b641028f0 2024-04-02T04:49:55.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.219+0000 7f7b68e73700 1 --2- 172.21.15.152:0/742064498 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7b641028f0 0x7f7b64111700 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f7b5c009fd0 tx=0x7f7b5c009840 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:55.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.219+0000 7f7b617fa700 1 -- 172.21.15.152:0/742064498 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7b5c0092e0 con 0x7f7b641028f0 2024-04-02T04:49:55.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.220+0000 7f7b617fa700 1 -- 172.21.15.152:0/742064498 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f7b5c007ce0 con 0x7f7b641028f0 2024-04-02T04:49:55.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.220+0000 7f7b6b0d7700 1 -- 172.21.15.152:0/742064498 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7b64111e40 con 0x7f7b641028f0 2024-04-02T04:49:55.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.220+0000 7f7b617fa700 1 -- 172.21.15.152:0/742064498 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7b5c025990 con 0x7f7b641028f0 2024-04-02T04:49:55.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.220+0000 7f7b6b0d7700 1 -- 172.21.15.152:0/742064498 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7b641129b0 con 0x7f7b641028f0 2024-04-02T04:49:55.877 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.220+0000 7f7b617fa700 1 -- 172.21.15.152:0/742064498 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 8) v1 ==== 43368+0+0 (secure 0 0 0) 0x7f7b5c019050 con 0x7f7b641028f0 2024-04-02T04:49:55.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.220+0000 7f7b6b0d7700 1 -- 172.21.15.152:0/742064498 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7b6405f5f0 con 0x7f7b641028f0 2024-04-02T04:49:55.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.221+0000 7f7b617fa700 1 --2- 172.21.15.152:0/742064498 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f7b500367e0 0x7f7b50038c90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:55.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.221+0000 7f7b617fa700 1 -- 172.21.15.152:0/742064498 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 936+0+0 (secure 0 0 0) 0x7f7b5c04a070 con 0x7f7b641028f0 2024-04-02T04:49:55.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.221+0000 7f7b63fff700 1 --2- 172.21.15.152:0/742064498 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f7b500367e0 0x7f7b50038c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:55.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.221+0000 7f7b63fff700 1 --2- 172.21.15.152:0/742064498 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f7b500367e0 0x7f7b50038c90 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f7b54009990 tx=0x7f7b54006e40 comp rx=0 tx=0).ready entity=mgr.14118 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:55.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.225+0000 7f7b617fa700 1 -- 172.21.15.152:0/742064498 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f7b5c02d9a0 con 0x7f7b641028f0 2024-04-02T04:49:55.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:54.464+0000 7f7b6b0d7700 1 -- 172.21.15.152:0/742064498 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "mgr module enable", "module": "dashboard"} v 0) v1 -- 0x7f7b6404bed0 con 0x7f7b641028f0 2024-04-02T04:49:55.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:55.284+0000 7f7b617fa700 1 -- 172.21.15.152:0/742064498 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mgrmap(e 9) v1 ==== 43381+0+0 (secure 0 0 0) 0x7f7b5c022d70 con 0x7f7b641028f0 2024-04-02T04:49:55.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:55.285+0000 7f7b617fa700 1 -- 172.21.15.152:0/742064498 <== mon.0 v2:172.21.15.152:3300/0 8 ==== mon_command_ack([{"prefix": "mgr module enable", "module": "dashboard"}]=0 v9) v1 ==== 88+0+0 (secure 0 0 0) 0x7f7b5c0245a0 con 0x7f7b641028f0 2024-04-02T04:49:55.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:55.288+0000 7f7b6b0d7700 1 -- 172.21.15.152:0/742064498 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f7b500367e0 msgr2=0x7f7b50038c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:55.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:55.288+0000 7f7b6b0d7700 1 --2- 172.21.15.152:0/742064498 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f7b500367e0 0x7f7b50038c90 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7f7b54009990 tx=0x7f7b54006e40 comp rx=0 tx=0).stop 2024-04-02T04:49:55.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:55.288+0000 7f7b6b0d7700 1 -- 172.21.15.152:0/742064498 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7b641028f0 msgr2=0x7f7b64111700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:55.878 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:55.288+0000 7f7b6b0d7700 1 --2- 172.21.15.152:0/742064498 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7b641028f0 0x7f7b64111700 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f7b5c009fd0 tx=0x7f7b5c009840 comp rx=0 tx=0).stop 2024-04-02T04:49:55.879 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:55.288+0000 7f7b6b0d7700 1 -- 172.21.15.152:0/742064498 shutdown_connections 2024-04-02T04:49:55.879 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:55.288+0000 7f7b6b0d7700 1 --2- 172.21.15.152:0/742064498 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f7b500367e0 0x7f7b50038c90 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:55.879 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:55.288+0000 7f7b6b0d7700 1 --2- 172.21.15.152:0/742064498 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7b641028f0 0x7f7b64111700 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:55.879 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:55.288+0000 7f7b6b0d7700 1 -- 172.21.15.152:0/742064498 >> 172.21.15.152:0/742064498 conn(0x7f7b640fe0d0 msgr2=0x7f7b640fec20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:55.879 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:55.288+0000 7f7b6b0d7700 1 -- 172.21.15.152:0/742064498 shutdown_connections 2024-04-02T04:49:55.879 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:55.288+0000 7f7b6b0d7700 1 -- 172.21.15.152:0/742064498 wait complete. 2024-04-02T04:49:56.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:56 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/742064498' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-02T04:49:56.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:56 smithi152 ceph-mon[29073]: mgrmap e9: smithi152.jivdcl(active, since 24s) 2024-04-02T04:49:57.267 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout { 2024-04-02T04:49:57.267 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-04-02T04:49:57.267 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-02T04:49:57.267 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "active_name": "smithi152.jivdcl", 2024-04-02T04:49:57.267 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-02T04:49:57.267 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout } 2024-04-02T04:49:57.267 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.356+0000 7f31fd198700 1 Processor -- start 2024-04-02T04:49:57.267 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.356+0000 7f31fd198700 1 -- start start 2024-04-02T04:49:57.268 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.356+0000 7f31fd198700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f31f81028c0 0x7f31f8102cd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:57.268 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.356+0000 7f31fd198700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f31f8103210 con 0x7f31f81028c0 2024-04-02T04:49:57.268 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.356+0000 7f31f6d9d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f31f81028c0 0x7f31f8102cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:57.268 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.356+0000 7f31f6d9d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f31f81028c0 0x7f31f8102cd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:44182/0 (socket says 172.21.15.152:44182) 2024-04-02T04:49:57.268 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.356+0000 7f31f6d9d700 1 -- 172.21.15.152:0/165037772 learned_addr learned my addr 172.21.15.152:0/165037772 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:57.268 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.356+0000 7f31f6d9d700 1 -- 172.21.15.152:0/165037772 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f31f8103350 con 0x7f31f81028c0 2024-04-02T04:49:57.268 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.357+0000 7f31f6d9d700 1 --2- 172.21.15.152:0/165037772 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f31f81028c0 0x7f31f8102cd0 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f31e80098e0 tx=0x7f31e8009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=80b521de64b2b3bc server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:57.268 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.357+0000 7f31f5d9b700 1 -- 172.21.15.152:0/165037772 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f31e800b860 con 0x7f31f81028c0 2024-04-02T04:49:57.268 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.357+0000 7f31f5d9b700 1 -- 172.21.15.152:0/165037772 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f31e800be90 con 0x7f31f81028c0 2024-04-02T04:49:57.268 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.357+0000 7f31f5d9b700 1 -- 172.21.15.152:0/165037772 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f31e80103c0 con 0x7f31f81028c0 2024-04-02T04:49:57.268 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.357+0000 7f31fd198700 1 -- 172.21.15.152:0/165037772 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f31f81028c0 msgr2=0x7f31f8102cd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:57.268 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.358+0000 7f31fd198700 1 --2- 172.21.15.152:0/165037772 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f31f81028c0 0x7f31f8102cd0 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7f31e80098e0 tx=0x7f31e8009250 comp rx=0 tx=0).stop 2024-04-02T04:49:57.269 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.358+0000 7f31fd198700 1 -- 172.21.15.152:0/165037772 shutdown_connections 2024-04-02T04:49:57.269 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.358+0000 7f31fd198700 1 --2- 172.21.15.152:0/165037772 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f31f81028c0 0x7f31f8102cd0 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:57.269 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.358+0000 7f31fd198700 1 -- 172.21.15.152:0/165037772 >> 172.21.15.152:0/165037772 conn(0x7f31f80fe080 msgr2=0x7f31f8100490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:57.269 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.358+0000 7f31fd198700 1 -- 172.21.15.152:0/165037772 shutdown_connections 2024-04-02T04:49:57.269 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.358+0000 7f31fd198700 1 -- 172.21.15.152:0/165037772 wait complete. 2024-04-02T04:49:57.269 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.358+0000 7f31fd198700 1 Processor -- start 2024-04-02T04:49:57.269 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.358+0000 7f31fd198700 1 -- start start 2024-04-02T04:49:57.269 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.358+0000 7f31fd198700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f31f81028c0 0x7f31f8078c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:57.269 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.359+0000 7f31fd198700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f31f8079180 con 0x7f31f81028c0 2024-04-02T04:49:57.269 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.359+0000 7f31f6d9d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f31f81028c0 0x7f31f8078c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:49:57.269 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.359+0000 7f31f6d9d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f31f81028c0 0x7f31f8078c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:44186/0 (socket says 172.21.15.152:44186) 2024-04-02T04:49:57.269 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.359+0000 7f31f6d9d700 1 -- 172.21.15.152:0/2283937445 learned_addr learned my addr 172.21.15.152:0/2283937445 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:49:57.270 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.359+0000 7f31f6d9d700 1 -- 172.21.15.152:0/2283937445 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f31e80094f0 con 0x7f31f81028c0 2024-04-02T04:49:57.270 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.359+0000 7f31f6d9d700 1 --2- 172.21.15.152:0/2283937445 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f31f81028c0 0x7f31f8078c40 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f31e8011680 tx=0x7f31e8009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:49:57.270 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.359+0000 7f31df7fe700 1 -- 172.21.15.152:0/2283937445 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f31e8010890 con 0x7f31f81028c0 2024-04-02T04:49:57.270 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.359+0000 7f31df7fe700 1 -- 172.21.15.152:0/2283937445 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f31e8010ec0 con 0x7f31f81028c0 2024-04-02T04:49:57.270 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.359+0000 7f31df7fe700 1 -- 172.21.15.152:0/2283937445 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f31e80243f0 con 0x7f31f81028c0 2024-04-02T04:49:57.270 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.359+0000 7f31fd198700 1 -- 172.21.15.152:0/2283937445 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f31f80771f0 con 0x7f31f81028c0 2024-04-02T04:49:57.270 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.359+0000 7f31fd198700 1 -- 172.21.15.152:0/2283937445 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f31f80776e0 con 0x7f31f81028c0 2024-04-02T04:49:57.270 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.360+0000 7f31df7fe700 1 -- 172.21.15.152:0/2283937445 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 9) v1 ==== 43381+0+0 (secure 0 0 0) 0x7f31e802d3f0 con 0x7f31f81028c0 2024-04-02T04:49:57.270 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.360+0000 7f31fd198700 1 -- 172.21.15.152:0/2283937445 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f31f804bed0 con 0x7f31f81028c0 2024-04-02T04:49:57.270 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.361+0000 7f31df7fe700 1 --2- 172.21.15.152:0/2283937445 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f31e0036830 0x7f31e0038ce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:49:57.270 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.361+0000 7f31df7fe700 1 -- 172.21.15.152:0/2283937445 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 936+0+0 (secure 0 0 0) 0x7f31e804a280 con 0x7f31f81028c0 2024-04-02T04:49:57.271 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.361+0000 7f31f659c700 1 -- 172.21.15.152:0/2283937445 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f31e0036830 msgr2=0x7f31e0038ce0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.152:6800/3544077480 2024-04-02T04:49:57.271 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.361+0000 7f31f659c700 1 --2- 172.21.15.152:0/2283937445 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f31e0036830 0x7f31e0038ce0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-02T04:49:57.271 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.365+0000 7f31df7fe700 1 -- 172.21.15.152:0/2283937445 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f31e802b8e0 con 0x7f31f81028c0 2024-04-02T04:49:57.271 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.562+0000 7f31f659c700 1 -- 172.21.15.152:0/2283937445 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f31e0036830 msgr2=0x7f31e0038ce0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.152:6800/3544077480 2024-04-02T04:49:57.271 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.562+0000 7f31f659c700 1 --2- 172.21.15.152:0/2283937445 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f31e0036830 0x7f31e0038ce0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-04-02T04:49:57.271 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.609+0000 7f31fd198700 1 -- 172.21.15.152:0/2283937445 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "mgr stat"} v 0) v1 -- 0x7f31f8077d80 con 0x7f31f81028c0 2024-04-02T04:49:57.271 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.609+0000 7f31df7fe700 1 -- 172.21.15.152:0/2283937445 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "mgr stat"}]=0 v9) v1 ==== 56+0+103 (secure 0 0 0) 0x7f31e8006d70 con 0x7f31f81028c0 2024-04-02T04:49:57.271 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.613+0000 7f31fd198700 1 -- 172.21.15.152:0/2283937445 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f31e0036830 msgr2=0x7f31e0038ce0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-02T04:49:57.271 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.613+0000 7f31fd198700 1 --2- 172.21.15.152:0/2283937445 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f31e0036830 0x7f31e0038ce0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:57.271 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.613+0000 7f31fd198700 1 -- 172.21.15.152:0/2283937445 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f31f81028c0 msgr2=0x7f31f8078c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:49:57.271 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.613+0000 7f31fd198700 1 --2- 172.21.15.152:0/2283937445 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f31f81028c0 0x7f31f8078c40 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f31e8011680 tx=0x7f31e8009250 comp rx=0 tx=0).stop 2024-04-02T04:49:57.271 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.613+0000 7f31fd198700 1 -- 172.21.15.152:0/2283937445 shutdown_connections 2024-04-02T04:49:57.271 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.613+0000 7f31fd198700 1 --2- 172.21.15.152:0/2283937445 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f31e0036830 0x7f31e0038ce0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:57.272 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.613+0000 7f31fd198700 1 --2- 172.21.15.152:0/2283937445 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f31f81028c0 0x7f31f8078c40 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:49:57.272 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.613+0000 7f31fd198700 1 -- 172.21.15.152:0/2283937445 >> 172.21.15.152:0/2283937445 conn(0x7f31f80fe080 msgr2=0x7f31f80febf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:49:57.272 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.613+0000 7f31fd198700 1 -- 172.21.15.152:0/2283937445 shutdown_connections 2024-04-02T04:49:57.272 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:56.613+0000 7f31fd198700 1 -- 172.21.15.152:0/2283937445 wait complete. 2024-04-02T04:49:57.272 INFO:teuthology.orchestra.run.smithi152.stdout:Waiting for the mgr to restart... 2024-04-02T04:49:57.272 INFO:teuthology.orchestra.run.smithi152.stdout:Waiting for mgr epoch 9... 2024-04-02T04:49:57.429 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:49:57 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/2283937445' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-02T04:50:02.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:01 smithi152 ceph-mon[29073]: Active manager daemon smithi152.jivdcl restarted 2024-04-02T04:50:02.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:01 smithi152 ceph-mon[29073]: Activating manager daemon smithi152.jivdcl 2024-04-02T04:50:02.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:01 smithi152 ceph-mon[29073]: osdmap e3: 0 total, 0 up, 0 in 2024-04-02T04:50:02.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:01 smithi152 ceph-mon[29073]: mgrmap e10: smithi152.jivdcl(active, starting, since 0.00328031s) 2024-04-02T04:50:02.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:01 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi152"}]: dispatch 2024-04-02T04:50:02.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:01 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mgr metadata", "who": "smithi152.jivdcl", "id": "smithi152.jivdcl"}]: dispatch 2024-04-02T04:50:02.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:01 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T04:50:02.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:01 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T04:50:02.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:01 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T04:50:02.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:01 smithi152 ceph-mon[29073]: Manager daemon smithi152.jivdcl is now available 2024-04-02T04:50:02.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:01 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:50:02.008 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:01 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:50:02.008 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:01 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi152.jivdcl/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:50:02.008 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:01 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi152.jivdcl/trash_purge_schedule"}]: dispatch 2024-04-02T04:50:03.487 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout { 2024-04-02T04:50:03.488 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-04-02T04:50:03.488 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-02T04:50:03.488 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout } 2024-04-02T04:50:03.488 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.672+0000 7f271e0c9700 1 Processor -- start 2024-04-02T04:50:03.488 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.672+0000 7f271e0c9700 1 -- start start 2024-04-02T04:50:03.488 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.672+0000 7f271e0c9700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f27181028c0 0x7f2718102cd0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:03.488 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.672+0000 7f271e0c9700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2718103210 con 0x7f27181028c0 2024-04-02T04:50:03.488 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.673+0000 7f27177fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f27181028c0 0x7f2718102cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:03.488 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.673+0000 7f27177fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f27181028c0 0x7f2718102cd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:44194/0 (socket says 172.21.15.152:44194) 2024-04-02T04:50:03.488 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.673+0000 7f27177fe700 1 -- 172.21.15.152:0/369993255 learned_addr learned my addr 172.21.15.152:0/369993255 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:50:03.488 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.673+0000 7f27177fe700 1 -- 172.21.15.152:0/369993255 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2718103350 con 0x7f27181028c0 2024-04-02T04:50:03.489 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.673+0000 7f27177fe700 1 --2- 172.21.15.152:0/369993255 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f27181028c0 0x7f2718102cd0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f27100098e0 tx=0x7f2710009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=2dcdba5a03d49e4c server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:03.489 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.673+0000 7f27167fc700 1 -- 172.21.15.152:0/369993255 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f271000b860 con 0x7f27181028c0 2024-04-02T04:50:03.489 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.673+0000 7f27167fc700 1 -- 172.21.15.152:0/369993255 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f271000be90 con 0x7f27181028c0 2024-04-02T04:50:03.489 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.673+0000 7f27167fc700 1 -- 172.21.15.152:0/369993255 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f27100103c0 con 0x7f27181028c0 2024-04-02T04:50:03.489 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.674+0000 7f271e0c9700 1 -- 172.21.15.152:0/369993255 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f27181028c0 msgr2=0x7f2718102cd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:03.489 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.674+0000 7f271e0c9700 1 --2- 172.21.15.152:0/369993255 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f27181028c0 0x7f2718102cd0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f27100098e0 tx=0x7f2710009250 comp rx=0 tx=0).stop 2024-04-02T04:50:03.489 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.674+0000 7f271e0c9700 1 -- 172.21.15.152:0/369993255 shutdown_connections 2024-04-02T04:50:03.489 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.674+0000 7f271e0c9700 1 --2- 172.21.15.152:0/369993255 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f27181028c0 0x7f2718102cd0 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:03.489 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.674+0000 7f271e0c9700 1 -- 172.21.15.152:0/369993255 >> 172.21.15.152:0/369993255 conn(0x7f27180fe080 msgr2=0x7f2718100490 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:03.489 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.674+0000 7f271e0c9700 1 -- 172.21.15.152:0/369993255 shutdown_connections 2024-04-02T04:50:03.489 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.674+0000 7f271e0c9700 1 -- 172.21.15.152:0/369993255 wait complete. 2024-04-02T04:50:03.489 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.675+0000 7f271e0c9700 1 Processor -- start 2024-04-02T04:50:03.489 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.675+0000 7f271e0c9700 1 -- start start 2024-04-02T04:50:03.489 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.675+0000 7f271e0c9700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f27181028c0 0x7f2718078c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:03.489 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.675+0000 7f271e0c9700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2718079180 con 0x7f27181028c0 2024-04-02T04:50:03.490 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.675+0000 7f27177fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f27181028c0 0x7f2718078c40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:03.490 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.675+0000 7f27177fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f27181028c0 0x7f2718078c40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:44208/0 (socket says 172.21.15.152:44208) 2024-04-02T04:50:03.490 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.675+0000 7f27177fe700 1 -- 172.21.15.152:0/562537194 learned_addr learned my addr 172.21.15.152:0/562537194 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:50:03.490 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.675+0000 7f27177fe700 1 -- 172.21.15.152:0/562537194 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f27100094f0 con 0x7f27181028c0 2024-04-02T04:50:03.490 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.676+0000 7f27177fe700 1 --2- 172.21.15.152:0/562537194 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f27181028c0 0x7f2718078c40 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f27100090b0 tx=0x7f2710009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:03.490 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.676+0000 7f26fffff700 1 -- 172.21.15.152:0/562537194 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f2710010840 con 0x7f27181028c0 2024-04-02T04:50:03.490 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.676+0000 7f26fffff700 1 -- 172.21.15.152:0/562537194 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f2710010e70 con 0x7f27181028c0 2024-04-02T04:50:03.490 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.676+0000 7f271e0c9700 1 -- 172.21.15.152:0/562537194 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f27180771f0 con 0x7f27181028c0 2024-04-02T04:50:03.490 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.676+0000 7f26fffff700 1 -- 172.21.15.152:0/562537194 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f2710024e40 con 0x7f27181028c0 2024-04-02T04:50:03.490 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.676+0000 7f271e0c9700 1 -- 172.21.15.152:0/562537194 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f27180776e0 con 0x7f27181028c0 2024-04-02T04:50:03.490 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.677+0000 7f26fffff700 1 -- 172.21.15.152:0/562537194 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 9) v1 ==== 43381+0+0 (secure 0 0 0) 0x7f271001a570 con 0x7f27181028c0 2024-04-02T04:50:03.490 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.677+0000 7f26fffff700 1 --2- 172.21.15.152:0/562537194 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f2700036790 0x7f2700038c40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:03.490 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.677+0000 7f26fffff700 1 -- 172.21.15.152:0/562537194 --> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x7f2700039350 con 0x7f2700036790 2024-04-02T04:50:03.491 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.677+0000 7f26fffff700 1 -- 172.21.15.152:0/562537194 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 936+0+0 (secure 0 0 0) 0x7f271004ab80 con 0x7f27181028c0 2024-04-02T04:50:03.491 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.677+0000 7f2716ffd700 1 -- 172.21.15.152:0/562537194 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f2700036790 msgr2=0x7f2700038c40 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.152:6800/3544077480 2024-04-02T04:50:03.491 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.677+0000 7f2716ffd700 1 --2- 172.21.15.152:0/562537194 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f2700036790 0x7f2700038c40 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-02T04:50:03.491 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.878+0000 7f2716ffd700 1 -- 172.21.15.152:0/562537194 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f2700036790 msgr2=0x7f2700038c40 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.152:6800/3544077480 2024-04-02T04:50:03.491 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:57.878+0000 7f2716ffd700 1 --2- 172.21.15.152:0/562537194 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f2700036790 0x7f2700038c40 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-04-02T04:50:03.491 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:58.279+0000 7f2716ffd700 1 -- 172.21.15.152:0/562537194 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f2700036790 msgr2=0x7f2700038c40 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.152:6800/3544077480 2024-04-02T04:50:03.491 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:58.279+0000 7f2716ffd700 1 --2- 172.21.15.152:0/562537194 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f2700036790 0x7f2700038c40 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.800000 2024-04-02T04:50:03.491 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:59.080+0000 7f2716ffd700 1 -- 172.21.15.152:0/562537194 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f2700036790 msgr2=0x7f2700038c40 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.152:6800/3544077480 2024-04-02T04:50:03.491 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:49:59.080+0000 7f2716ffd700 1 --2- 172.21.15.152:0/562537194 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f2700036790 0x7f2700038c40 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 1.600000 2024-04-02T04:50:03.491 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:00.681+0000 7f2716ffd700 1 -- 172.21.15.152:0/562537194 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f2700036790 msgr2=0x7f2700038c40 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.152:6800/3544077480 2024-04-02T04:50:03.491 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:00.681+0000 7f2716ffd700 1 --2- 172.21.15.152:0/562537194 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f2700036790 0x7f2700038c40 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 3.200000 2024-04-02T04:50:03.491 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:01.903+0000 7f26fffff700 1 -- 172.21.15.152:0/562537194 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mgrmap(e 10) v1 ==== 43148+0+0 (secure 0 0 0) 0x7f2710024430 con 0x7f27181028c0 2024-04-02T04:50:03.491 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:01.903+0000 7f26fffff700 1 -- 172.21.15.152:0/562537194 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f2700036790 msgr2=0x7f2700038c40 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-02T04:50:03.491 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:01.903+0000 7f26fffff700 1 --2- 172.21.15.152:0/562537194 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f2700036790 0x7f2700038c40 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:03.492 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:02.905+0000 7f26fffff700 1 -- 172.21.15.152:0/562537194 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mgrmap(e 11) v1 ==== 43275+0+0 (secure 0 0 0) 0x7f2710024710 con 0x7f27181028c0 2024-04-02T04:50:03.492 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:02.905+0000 7f26fffff700 1 --2- 172.21.15.152:0/562537194 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f270003a070 0x7f270003c450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:03.492 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:02.905+0000 7f26fffff700 1 -- 172.21.15.152:0/562537194 --> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x7f2700039350 con 0x7f270003a070 2024-04-02T04:50:03.492 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:02.905+0000 7f2716ffd700 1 --2- 172.21.15.152:0/562537194 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f270003a070 0x7f270003c450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:03.492 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:02.906+0000 7f2716ffd700 1 --2- 172.21.15.152:0/562537194 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f270003a070 0x7f270003c450 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f270c0039d0 tx=0x7f270c0092f0 comp rx=0 tx=0).ready entity=mgr.14160 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:03.492 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:02.906+0000 7f26fffff700 1 -- 172.21.15.152:0/562537194 <== mgr.14160 v2:172.21.15.152:6800/2497421829 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+6549 (secure 0 0 0) 0x7f2700039350 con 0x7f270003a070 2024-04-02T04:50:03.492 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:02.912+0000 7f271e0c9700 1 -- 172.21.15.152:0/562537194 --> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] -- command(tid 1: {"prefix": "mgr_status"}) v1 -- 0x7f271804bed0 con 0x7f270003a070 2024-04-02T04:50:03.492 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:02.913+0000 7f26fffff700 1 -- 172.21.15.152:0/562537194 <== mgr.14160 v2:172.21.15.152:6800/2497421829 2 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+52 (secure 0 0 0) 0x7f271804bed0 con 0x7f270003a070 2024-04-02T04:50:03.492 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:02.913+0000 7f271e0c9700 1 -- 172.21.15.152:0/562537194 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f270003a070 msgr2=0x7f270003c450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:03.492 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:02.913+0000 7f271e0c9700 1 --2- 172.21.15.152:0/562537194 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f270003a070 0x7f270003c450 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 crypto rx=0x7f270c0039d0 tx=0x7f270c0092f0 comp rx=0 tx=0).stop 2024-04-02T04:50:03.492 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:02.913+0000 7f271e0c9700 1 -- 172.21.15.152:0/562537194 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f27181028c0 msgr2=0x7f2718078c40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:03.492 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:02.913+0000 7f271e0c9700 1 --2- 172.21.15.152:0/562537194 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f27181028c0 0x7f2718078c40 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f27100090b0 tx=0x7f2710009250 comp rx=0 tx=0).stop 2024-04-02T04:50:03.492 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:02.913+0000 7f271e0c9700 1 -- 172.21.15.152:0/562537194 shutdown_connections 2024-04-02T04:50:03.492 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:02.913+0000 7f271e0c9700 1 --2- 172.21.15.152:0/562537194 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f270003a070 0x7f270003c450 unknown :-1 s=CLOSED pgs=1 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:03.492 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:02.913+0000 7f271e0c9700 1 --2- 172.21.15.152:0/562537194 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f27181028c0 0x7f2718078c40 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:03.493 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:02.913+0000 7f271e0c9700 1 --2- 172.21.15.152:0/562537194 >> [v2:172.21.15.152:6800/3544077480,v1:172.21.15.152:6801/3544077480] conn(0x7f2700036790 0x7f2700038c40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:03.493 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:02.913+0000 7f271e0c9700 1 -- 172.21.15.152:0/562537194 >> 172.21.15.152:0/562537194 conn(0x7f27180fe080 msgr2=0x7f27180febf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:03.493 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:02.913+0000 7f271e0c9700 1 -- 172.21.15.152:0/562537194 shutdown_connections 2024-04-02T04:50:03.493 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:02.913+0000 7f271e0c9700 1 -- 172.21.15.152:0/562537194 wait complete. 2024-04-02T04:50:03.493 INFO:teuthology.orchestra.run.smithi152.stdout:mgr epoch 9 is available 2024-04-02T04:50:03.493 INFO:teuthology.orchestra.run.smithi152.stdout:Generating a dashboard self-signed certificate... 2024-04-02T04:50:03.822 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:03 smithi152 ceph-mon[29073]: [02/Apr/2024:04:50:02] ENGINE Bus STARTING 2024-04-02T04:50:03.822 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:03 smithi152 ceph-mon[29073]: [02/Apr/2024:04:50:02] ENGINE Serving on https://172.21.15.152:7150 2024-04-02T04:50:03.822 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:03 smithi152 ceph-mon[29073]: [02/Apr/2024:04:50:02] ENGINE Bus STARTED 2024-04-02T04:50:03.822 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:03 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:03.822 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:03 smithi152 ceph-mon[29073]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-02T04:50:03.822 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:03 smithi152 ceph-mon[29073]: mgrmap e11: smithi152.jivdcl(active, since 1.006s) 2024-04-02T04:50:03.823 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:03 smithi152 ceph-mon[29073]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-02T04:50:03.823 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:03 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:04.717 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-04-02T04:50:04.717 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.915+0000 7fe7810dd700 1 Processor -- start 2024-04-02T04:50:04.717 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.915+0000 7fe7810dd700 1 -- start start 2024-04-02T04:50:04.717 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.915+0000 7fe7810dd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe77c101850 0x7fe77c101c60 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:04.718 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.915+0000 7fe7810dd700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe77c1021a0 con 0x7fe77c101850 2024-04-02T04:50:04.718 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.915+0000 7fe77ad9d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe77c101850 0x7fe77c101c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:04.718 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.915+0000 7fe77ad9d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe77c101850 0x7fe77c101c60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:34504/0 (socket says 172.21.15.152:34504) 2024-04-02T04:50:04.718 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.915+0000 7fe77ad9d700 1 -- 172.21.15.152:0/3003339821 learned_addr learned my addr 172.21.15.152:0/3003339821 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:50:04.718 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.915+0000 7fe77ad9d700 1 -- 172.21.15.152:0/3003339821 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe77c1022e0 con 0x7fe77c101850 2024-04-02T04:50:04.718 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.915+0000 7fe77ad9d700 1 --2- 172.21.15.152:0/3003339821 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe77c101850 0x7fe77c101c60 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7fe76c0098e0 tx=0x7fe76c009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=af1770e61b8f1546 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:04.718 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.916+0000 7fe779d9b700 1 -- 172.21.15.152:0/3003339821 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fe76c00b860 con 0x7fe77c101850 2024-04-02T04:50:04.718 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.916+0000 7fe779d9b700 1 -- 172.21.15.152:0/3003339821 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fe76c00be90 con 0x7fe77c101850 2024-04-02T04:50:04.718 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.916+0000 7fe779d9b700 1 -- 172.21.15.152:0/3003339821 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fe76c0103c0 con 0x7fe77c101850 2024-04-02T04:50:04.718 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.916+0000 7fe7810dd700 1 -- 172.21.15.152:0/3003339821 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe77c101850 msgr2=0x7fe77c101c60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:04.718 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.916+0000 7fe7810dd700 1 --2- 172.21.15.152:0/3003339821 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe77c101850 0x7fe77c101c60 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7fe76c0098e0 tx=0x7fe76c009250 comp rx=0 tx=0).stop 2024-04-02T04:50:04.719 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.916+0000 7fe7810dd700 1 -- 172.21.15.152:0/3003339821 shutdown_connections 2024-04-02T04:50:04.719 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.916+0000 7fe7810dd700 1 --2- 172.21.15.152:0/3003339821 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe77c101850 0x7fe77c101c60 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:04.719 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.916+0000 7fe7810dd700 1 -- 172.21.15.152:0/3003339821 >> 172.21.15.152:0/3003339821 conn(0x7fe77c0fcf90 msgr2=0x7fe77c0ff3e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:04.719 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.916+0000 7fe7810dd700 1 -- 172.21.15.152:0/3003339821 shutdown_connections 2024-04-02T04:50:04.719 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.916+0000 7fe7810dd700 1 -- 172.21.15.152:0/3003339821 wait complete. 2024-04-02T04:50:04.719 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.917+0000 7fe7810dd700 1 Processor -- start 2024-04-02T04:50:04.719 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.917+0000 7fe7810dd700 1 -- start start 2024-04-02T04:50:04.719 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.917+0000 7fe7810dd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe77c101850 0x7fe77c076b20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:04.719 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.917+0000 7fe7810dd700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe77c114440 con 0x7fe77c101850 2024-04-02T04:50:04.719 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.917+0000 7fe77ad9d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe77c101850 0x7fe77c076b20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:04.719 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.917+0000 7fe77ad9d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe77c101850 0x7fe77c076b20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:34520/0 (socket says 172.21.15.152:34520) 2024-04-02T04:50:04.719 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.917+0000 7fe77ad9d700 1 -- 172.21.15.152:0/3809203468 learned_addr learned my addr 172.21.15.152:0/3809203468 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:50:04.720 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.917+0000 7fe77ad9d700 1 -- 172.21.15.152:0/3809203468 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe76c0094f0 con 0x7fe77c101850 2024-04-02T04:50:04.720 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.918+0000 7fe77ad9d700 1 --2- 172.21.15.152:0/3809203468 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe77c101850 0x7fe77c076b20 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7fe76c011680 tx=0x7fe76c009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:04.720 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.918+0000 7fe7637fe700 1 -- 172.21.15.152:0/3809203468 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fe76c010890 con 0x7fe77c101850 2024-04-02T04:50:04.720 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.918+0000 7fe7637fe700 1 -- 172.21.15.152:0/3809203468 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fe76c010ec0 con 0x7fe77c101850 2024-04-02T04:50:04.720 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.918+0000 7fe7810dd700 1 -- 172.21.15.152:0/3809203468 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe77c114580 con 0x7fe77c101850 2024-04-02T04:50:04.720 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.918+0000 7fe7637fe700 1 -- 172.21.15.152:0/3809203468 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fe76c0243f0 con 0x7fe77c101850 2024-04-02T04:50:04.720 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.918+0000 7fe7810dd700 1 -- 172.21.15.152:0/3809203468 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe77c114a10 con 0x7fe77c101850 2024-04-02T04:50:04.720 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.919+0000 7fe7637fe700 1 -- 172.21.15.152:0/3809203468 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 11) v1 ==== 43275+0+0 (secure 0 0 0) 0x7fe76c02d3f0 con 0x7fe77c101850 2024-04-02T04:50:04.720 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.919+0000 7fe7637fe700 1 --2- 172.21.15.152:0/3809203468 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fe7640366b0 0x7fe764038b60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:04.720 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.919+0000 7fe7637fe700 1 -- 172.21.15.152:0/3809203468 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1151+0+0 (secure 0 0 0) 0x7fe76c04a1e0 con 0x7fe77c101850 2024-04-02T04:50:04.720 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.919+0000 7fe77a59c700 1 --2- 172.21.15.152:0/3809203468 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fe7640366b0 0x7fe764038b60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:04.720 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.919+0000 7fe7810dd700 1 -- 172.21.15.152:0/3809203468 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe77c04bed0 con 0x7fe77c101850 2024-04-02T04:50:04.721 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.919+0000 7fe77a59c700 1 --2- 172.21.15.152:0/3809203468 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fe7640366b0 0x7fe764038b60 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7fe770009990 tx=0x7fe770006e40 comp rx=0 tx=0).ready entity=mgr.14160 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:04.721 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:03.924+0000 7fe7637fe700 1 -- 172.21.15.152:0/3809203468 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fe76c02b8e0 con 0x7fe77c101850 2024-04-02T04:50:04.721 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:04.124+0000 7fe7810dd700 1 -- 172.21.15.152:0/3809203468 --> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] -- mgr_command(tid 0: {"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}) v1 -- 0x7fe77c0fe350 con 0x7fe7640366b0 2024-04-02T04:50:04.721 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:04.163+0000 7fe7637fe700 1 -- 172.21.15.152:0/3809203468 <== mgr.14160 v2:172.21.15.152:6800/2497421829 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+32 (secure 0 0 0) 0x7fe77c0fe350 con 0x7fe7640366b0 2024-04-02T04:50:04.721 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:04.165+0000 7fe7810dd700 1 -- 172.21.15.152:0/3809203468 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fe7640366b0 msgr2=0x7fe764038b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:04.721 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:04.165+0000 7fe7810dd700 1 --2- 172.21.15.152:0/3809203468 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fe7640366b0 0x7fe764038b60 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7fe770009990 tx=0x7fe770006e40 comp rx=0 tx=0).stop 2024-04-02T04:50:04.721 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:04.165+0000 7fe7810dd700 1 -- 172.21.15.152:0/3809203468 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe77c101850 msgr2=0x7fe77c076b20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:04.721 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:04.165+0000 7fe7810dd700 1 --2- 172.21.15.152:0/3809203468 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe77c101850 0x7fe77c076b20 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7fe76c011680 tx=0x7fe76c009250 comp rx=0 tx=0).stop 2024-04-02T04:50:04.721 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:04.166+0000 7fe7810dd700 1 -- 172.21.15.152:0/3809203468 shutdown_connections 2024-04-02T04:50:04.721 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:04.166+0000 7fe7810dd700 1 --2- 172.21.15.152:0/3809203468 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fe7640366b0 0x7fe764038b60 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:04.721 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:04.166+0000 7fe7810dd700 1 --2- 172.21.15.152:0/3809203468 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe77c101850 0x7fe77c076b20 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:04.721 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:04.166+0000 7fe7810dd700 1 -- 172.21.15.152:0/3809203468 >> 172.21.15.152:0/3809203468 conn(0x7fe77c0fcf90 msgr2=0x7fe77c0fdb80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:04.721 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:04.166+0000 7fe7810dd700 1 -- 172.21.15.152:0/3809203468 shutdown_connections 2024-04-02T04:50:04.722 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:04.166+0000 7fe7810dd700 1 -- 172.21.15.152:0/3809203468 wait complete. 2024-04-02T04:50:04.722 INFO:teuthology.orchestra.run.smithi152.stdout:Creating initial admin user... 2024-04-02T04:50:05.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:04 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:05.258 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:04 smithi152 ceph-mon[29073]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:50:05.258 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:04 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:05.258 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:04 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:05.258 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:04 smithi152 ceph-mon[29073]: mgrmap e12: smithi152.jivdcl(active, since 2s) 2024-04-02T04:50:06.590 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$nsekSa1Ig89.Aj5wFfdZfuA6C1GiaWDqvyPicULrVeXWdJN9v4.Je", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1712033405, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-02T04:50:06.590 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.133+0000 7ff3a0cbc700 1 Processor -- start 2024-04-02T04:50:06.590 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.133+0000 7ff3a0cbc700 1 -- start start 2024-04-02T04:50:06.590 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.133+0000 7ff3a0cbc700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff39c101d30 0x7ff39c102140 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:06.590 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.133+0000 7ff3a0cbc700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff39c102680 con 0x7ff39c101d30 2024-04-02T04:50:06.591 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.133+0000 7ff39a59c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff39c101d30 0x7ff39c102140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:06.591 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.133+0000 7ff39a59c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff39c101d30 0x7ff39c102140 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:34528/0 (socket says 172.21.15.152:34528) 2024-04-02T04:50:06.591 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.133+0000 7ff39a59c700 1 -- 172.21.15.152:0/1708123627 learned_addr learned my addr 172.21.15.152:0/1708123627 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:50:06.591 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.134+0000 7ff39a59c700 1 -- 172.21.15.152:0/1708123627 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff39c1027c0 con 0x7ff39c101d30 2024-04-02T04:50:06.591 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.134+0000 7ff39a59c700 1 --2- 172.21.15.152:0/1708123627 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff39c101d30 0x7ff39c102140 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7ff38c0098e0 tx=0x7ff38c009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=c8cf9cfc495bb350 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:06.591 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.134+0000 7ff399d9b700 1 -- 172.21.15.152:0/1708123627 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ff38c00b860 con 0x7ff39c101d30 2024-04-02T04:50:06.591 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.134+0000 7ff399d9b700 1 -- 172.21.15.152:0/1708123627 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7ff38c00be90 con 0x7ff39c101d30 2024-04-02T04:50:06.591 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.134+0000 7ff399d9b700 1 -- 172.21.15.152:0/1708123627 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ff38c0103c0 con 0x7ff39c101d30 2024-04-02T04:50:06.591 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.135+0000 7ff3a0cbc700 1 -- 172.21.15.152:0/1708123627 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff39c101d30 msgr2=0x7ff39c102140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:06.591 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.135+0000 7ff3a0cbc700 1 --2- 172.21.15.152:0/1708123627 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff39c101d30 0x7ff39c102140 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7ff38c0098e0 tx=0x7ff38c009250 comp rx=0 tx=0).stop 2024-04-02T04:50:06.591 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.135+0000 7ff3a0cbc700 1 -- 172.21.15.152:0/1708123627 shutdown_connections 2024-04-02T04:50:06.591 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.135+0000 7ff3a0cbc700 1 --2- 172.21.15.152:0/1708123627 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff39c101d30 0x7ff39c102140 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:06.591 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.135+0000 7ff3a0cbc700 1 -- 172.21.15.152:0/1708123627 >> 172.21.15.152:0/1708123627 conn(0x7ff39c0fc6c0 msgr2=0x7ff39c0fead0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:06.591 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.135+0000 7ff3a0cbc700 1 -- 172.21.15.152:0/1708123627 shutdown_connections 2024-04-02T04:50:06.592 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.135+0000 7ff3a0cbc700 1 -- 172.21.15.152:0/1708123627 wait complete. 2024-04-02T04:50:06.592 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.135+0000 7ff3a0cbc700 1 Processor -- start 2024-04-02T04:50:06.592 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.135+0000 7ff3a0cbc700 1 -- start start 2024-04-02T04:50:06.592 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.135+0000 7ff3a0cbc700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff39c101d30 0x7ff39c110610 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:06.592 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.135+0000 7ff3a0cbc700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff39c110b50 con 0x7ff39c101d30 2024-04-02T04:50:06.592 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.136+0000 7ff39a59c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff39c101d30 0x7ff39c110610 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:06.592 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.136+0000 7ff39a59c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff39c101d30 0x7ff39c110610 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:34540/0 (socket says 172.21.15.152:34540) 2024-04-02T04:50:06.592 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.136+0000 7ff39a59c700 1 -- 172.21.15.152:0/2039699839 learned_addr learned my addr 172.21.15.152:0/2039699839 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:50:06.592 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.136+0000 7ff39a59c700 1 -- 172.21.15.152:0/2039699839 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff38c0094f0 con 0x7ff39c101d30 2024-04-02T04:50:06.592 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.136+0000 7ff39a59c700 1 --2- 172.21.15.152:0/2039699839 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff39c101d30 0x7ff39c110610 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7ff38c006b00 tx=0x7ff38c009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:06.592 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.136+0000 7ff3937fe700 1 -- 172.21.15.152:0/2039699839 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ff38c010840 con 0x7ff39c101d30 2024-04-02T04:50:06.592 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.136+0000 7ff3937fe700 1 -- 172.21.15.152:0/2039699839 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7ff38c010e70 con 0x7ff39c101d30 2024-04-02T04:50:06.592 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.136+0000 7ff3a0cbc700 1 -- 172.21.15.152:0/2039699839 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff39c110d50 con 0x7ff39c101d30 2024-04-02T04:50:06.592 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.136+0000 7ff3937fe700 1 -- 172.21.15.152:0/2039699839 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ff38c01ae70 con 0x7ff39c101d30 2024-04-02T04:50:06.592 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.136+0000 7ff3a0cbc700 1 -- 172.21.15.152:0/2039699839 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff39c1118c0 con 0x7ff39c101d30 2024-04-02T04:50:06.593 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.137+0000 7ff3937fe700 1 -- 172.21.15.152:0/2039699839 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 12) v1 ==== 43381+0+0 (secure 0 0 0) 0x7ff38c0109a0 con 0x7ff39c101d30 2024-04-02T04:50:06.593 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.137+0000 7ff3a0cbc700 1 -- 172.21.15.152:0/2039699839 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff39c05f5f0 con 0x7ff39c101d30 2024-04-02T04:50:06.593 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.137+0000 7ff3937fe700 1 --2- 172.21.15.152:0/2039699839 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7ff3880367e0 0x7ff388038c90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:06.593 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.137+0000 7ff3937fe700 1 -- 172.21.15.152:0/2039699839 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1151+0+0 (secure 0 0 0) 0x7ff38c04ac60 con 0x7ff39c101d30 2024-04-02T04:50:06.593 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.137+0000 7ff391d9b700 1 --2- 172.21.15.152:0/2039699839 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7ff3880367e0 0x7ff388038c90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:06.593 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.138+0000 7ff391d9b700 1 --2- 172.21.15.152:0/2039699839 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7ff3880367e0 0x7ff388038c90 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7ff384009990 tx=0x7ff384006e40 comp rx=0 tx=0).ready entity=mgr.14160 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:06.593 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.142+0000 7ff3937fe700 1 -- 172.21.15.152:0/2039699839 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7ff38c01a430 con 0x7ff39c101d30 2024-04-02T04:50:06.593 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.368+0000 7ff3a0cbc700 1 -- 172.21.15.152:0/2039699839 --> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] -- mgr_command(tid 0: {"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}) v1 -- 0x7ff39c05f410 con 0x7ff3880367e0 2024-04-02T04:50:06.593 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.592+0000 7ff3937fe700 1 -- 172.21.15.152:0/2039699839 <== mgr.14160 v2:172.21.15.152:6800/2497421829 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+252 (secure 0 0 0) 0x7ff39c05f410 con 0x7ff3880367e0 2024-04-02T04:50:06.593 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.594+0000 7ff3a0cbc700 1 -- 172.21.15.152:0/2039699839 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7ff3880367e0 msgr2=0x7ff388038c90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:06.593 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.594+0000 7ff3a0cbc700 1 --2- 172.21.15.152:0/2039699839 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7ff3880367e0 0x7ff388038c90 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7ff384009990 tx=0x7ff384006e40 comp rx=0 tx=0).stop 2024-04-02T04:50:06.593 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.594+0000 7ff3a0cbc700 1 -- 172.21.15.152:0/2039699839 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff39c101d30 msgr2=0x7ff39c110610 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:06.593 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.595+0000 7ff3a0cbc700 1 --2- 172.21.15.152:0/2039699839 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff39c101d30 0x7ff39c110610 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7ff38c006b00 tx=0x7ff38c009250 comp rx=0 tx=0).stop 2024-04-02T04:50:06.593 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.595+0000 7ff3a0cbc700 1 -- 172.21.15.152:0/2039699839 shutdown_connections 2024-04-02T04:50:06.593 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.595+0000 7ff3a0cbc700 1 --2- 172.21.15.152:0/2039699839 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7ff3880367e0 0x7ff388038c90 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:06.593 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.595+0000 7ff3a0cbc700 1 --2- 172.21.15.152:0/2039699839 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff39c101d30 0x7ff39c110610 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:06.594 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.595+0000 7ff3a0cbc700 1 -- 172.21.15.152:0/2039699839 >> 172.21.15.152:0/2039699839 conn(0x7ff39c0fc6c0 msgr2=0x7ff39c0fd260 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:06.594 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.595+0000 7ff3a0cbc700 1 -- 172.21.15.152:0/2039699839 shutdown_connections 2024-04-02T04:50:06.594 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:05.595+0000 7ff3a0cbc700 1 -- 172.21.15.152:0/2039699839 wait complete. 2024-04-02T04:50:06.594 INFO:teuthology.orchestra.run.smithi152.stdout:Fetching dashboard port number... 2024-04-02T04:50:06.885 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:06 smithi152 ceph-mon[29073]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:50:06.885 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:06 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:07.814 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stdout 8443 2024-04-02T04:50:07.814 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.991+0000 7f162cba2700 1 Processor -- start 2024-04-02T04:50:07.814 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.991+0000 7f162cba2700 1 -- start start 2024-04-02T04:50:07.814 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.991+0000 7f162cba2700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f16281017f0 0x7f1628101c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:07.814 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.991+0000 7f162cba2700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1628102140 con 0x7f16281017f0 2024-04-02T04:50:07.814 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.992+0000 7f162659c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f16281017f0 0x7f1628101c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:07.814 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.992+0000 7f162659c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f16281017f0 0x7f1628101c00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:34544/0 (socket says 172.21.15.152:34544) 2024-04-02T04:50:07.814 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.992+0000 7f162659c700 1 -- 172.21.15.152:0/2711556101 learned_addr learned my addr 172.21.15.152:0/2711556101 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:50:07.815 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.992+0000 7f162659c700 1 -- 172.21.15.152:0/2711556101 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1628102280 con 0x7f16281017f0 2024-04-02T04:50:07.815 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.992+0000 7f162659c700 1 --2- 172.21.15.152:0/2711556101 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f16281017f0 0x7f1628101c00 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f161800bb80 tx=0x7f161800b4f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=6cb41bfc1663c297 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:07.815 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.992+0000 7f162559a700 1 -- 172.21.15.152:0/2711556101 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f161800dd00 con 0x7f16281017f0 2024-04-02T04:50:07.815 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.992+0000 7f162559a700 1 -- 172.21.15.152:0/2711556101 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f161800de60 con 0x7f16281017f0 2024-04-02T04:50:07.815 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.992+0000 7f162559a700 1 -- 172.21.15.152:0/2711556101 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1618018920 con 0x7f16281017f0 2024-04-02T04:50:07.815 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.993+0000 7f162cba2700 1 -- 172.21.15.152:0/2711556101 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f16281017f0 msgr2=0x7f1628101c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:07.815 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.993+0000 7f162cba2700 1 --2- 172.21.15.152:0/2711556101 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f16281017f0 0x7f1628101c00 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7f161800bb80 tx=0x7f161800b4f0 comp rx=0 tx=0).stop 2024-04-02T04:50:07.815 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.993+0000 7f162cba2700 1 -- 172.21.15.152:0/2711556101 shutdown_connections 2024-04-02T04:50:07.815 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.993+0000 7f162cba2700 1 --2- 172.21.15.152:0/2711556101 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f16281017f0 0x7f1628101c00 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:07.815 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.993+0000 7f162cba2700 1 -- 172.21.15.152:0/2711556101 >> 172.21.15.152:0/2711556101 conn(0x7f16280fcfb0 msgr2=0x7f16280ff3c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:07.815 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.993+0000 7f162cba2700 1 -- 172.21.15.152:0/2711556101 shutdown_connections 2024-04-02T04:50:07.815 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.993+0000 7f162cba2700 1 -- 172.21.15.152:0/2711556101 wait complete. 2024-04-02T04:50:07.815 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.994+0000 7f162cba2700 1 Processor -- start 2024-04-02T04:50:07.815 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.994+0000 7f162cba2700 1 -- start start 2024-04-02T04:50:07.815 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.994+0000 7f162cba2700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f16281017f0 0x7f16281138c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:07.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.994+0000 7f162cba2700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1628113e00 con 0x7f16281017f0 2024-04-02T04:50:07.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.994+0000 7f162659c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f16281017f0 0x7f16281138c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:07.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.994+0000 7f162659c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f16281017f0 0x7f16281138c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:34558/0 (socket says 172.21.15.152:34558) 2024-04-02T04:50:07.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.994+0000 7f162659c700 1 -- 172.21.15.152:0/121925111 learned_addr learned my addr 172.21.15.152:0/121925111 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:50:07.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.994+0000 7f162659c700 1 -- 172.21.15.152:0/121925111 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f161800b790 con 0x7f16281017f0 2024-04-02T04:50:07.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.994+0000 7f162659c700 1 --2- 172.21.15.152:0/121925111 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f16281017f0 0x7f16281138c0 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f16180106a0 tx=0x7f161800b4f0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:07.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.995+0000 7f1616ffd700 1 -- 172.21.15.152:0/121925111 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1618018e90 con 0x7f16281017f0 2024-04-02T04:50:07.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.995+0000 7f1616ffd700 1 -- 172.21.15.152:0/121925111 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f161800dc70 con 0x7f16281017f0 2024-04-02T04:50:07.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.995+0000 7f162cba2700 1 -- 172.21.15.152:0/121925111 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1628114000 con 0x7f16281017f0 2024-04-02T04:50:07.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.995+0000 7f1616ffd700 1 -- 172.21.15.152:0/121925111 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1618021550 con 0x7f16281017f0 2024-04-02T04:50:07.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.995+0000 7f1616ffd700 1 -- 172.21.15.152:0/121925111 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 12) v1 ==== 43381+0+0 (secure 0 0 0) 0x7f1618021780 con 0x7f16281017f0 2024-04-02T04:50:07.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.995+0000 7f162cba2700 1 -- 172.21.15.152:0/121925111 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1628114b70 con 0x7f16281017f0 2024-04-02T04:50:07.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.996+0000 7f1616ffd700 1 --2- 172.21.15.152:0/121925111 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f1610036830 0x7f1610038ce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:07.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.996+0000 7f1616ffd700 1 -- 172.21.15.152:0/121925111 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1151+0+0 (secure 0 0 0) 0x7f1618014070 con 0x7f16281017f0 2024-04-02T04:50:07.816 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.996+0000 7f1625d9b700 1 --2- 172.21.15.152:0/121925111 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f1610036830 0x7f1610038ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:07.817 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.996+0000 7f1625d9b700 1 --2- 172.21.15.152:0/121925111 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f1610036830 0x7f1610038ce0 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f161c009990 tx=0x7f161c006e40 comp rx=0 tx=0).ready entity=mgr.14160 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:07.817 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:06.996+0000 7f162cba2700 1 -- 172.21.15.152:0/121925111 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f162805f5f0 con 0x7f16281017f0 2024-04-02T04:50:07.817 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:07.001+0000 7f1616ffd700 1 -- 172.21.15.152:0/121925111 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f161800edc0 con 0x7f16281017f0 2024-04-02T04:50:07.817 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:07.193+0000 7f162cba2700 1 -- 172.21.15.152:0/121925111 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"} v 0) v1 -- 0x7f1628103540 con 0x7f16281017f0 2024-04-02T04:50:07.817 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:07.193+0000 7f1616ffd700 1 -- 172.21.15.152:0/121925111 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]=0 v8) v1 ==== 112+0+5 (secure 0 0 0) 0x7f16180490b0 con 0x7f16281017f0 2024-04-02T04:50:07.817 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:07.196+0000 7f162cba2700 1 -- 172.21.15.152:0/121925111 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f1610036830 msgr2=0x7f1610038ce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:07.817 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:07.196+0000 7f162cba2700 1 --2- 172.21.15.152:0/121925111 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f1610036830 0x7f1610038ce0 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f161c009990 tx=0x7f161c006e40 comp rx=0 tx=0).stop 2024-04-02T04:50:07.817 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:07.196+0000 7f162cba2700 1 -- 172.21.15.152:0/121925111 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f16281017f0 msgr2=0x7f16281138c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:07.817 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:07.196+0000 7f162cba2700 1 --2- 172.21.15.152:0/121925111 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f16281017f0 0x7f16281138c0 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f16180106a0 tx=0x7f161800b4f0 comp rx=0 tx=0).stop 2024-04-02T04:50:07.817 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:07.196+0000 7f162cba2700 1 -- 172.21.15.152:0/121925111 shutdown_connections 2024-04-02T04:50:07.817 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:07.196+0000 7f162cba2700 1 --2- 172.21.15.152:0/121925111 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f1610036830 0x7f1610038ce0 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:07.817 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:07.196+0000 7f162cba2700 1 --2- 172.21.15.152:0/121925111 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f16281017f0 0x7f16281138c0 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:07.817 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:07.196+0000 7f162cba2700 1 -- 172.21.15.152:0/121925111 >> 172.21.15.152:0/121925111 conn(0x7f16280fcfb0 msgr2=0x7f1628105ca0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:07.818 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:07.196+0000 7f162cba2700 1 -- 172.21.15.152:0/121925111 shutdown_connections 2024-04-02T04:50:07.818 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:07.196+0000 7f162cba2700 1 -- 172.21.15.152:0/121925111 wait complete. 2024-04-02T04:50:07.820 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-04-02T04:50:07.821 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout disabled 2024-04-02T04:50:07.828 INFO:teuthology.orchestra.run.smithi152.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-04-02T04:50:07.828 INFO:teuthology.orchestra.run.smithi152.stdout:systemctl: stdout inactive 2024-04-02T04:50:07.828 INFO:teuthology.orchestra.run.smithi152.stdout:firewalld.service is not enabled 2024-04-02T04:50:07.828 INFO:teuthology.orchestra.run.smithi152.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-02T04:50:07.829 INFO:teuthology.orchestra.run.smithi152.stdout:Ceph Dashboard is now available at: 2024-04-02T04:50:07.830 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:50:07.830 INFO:teuthology.orchestra.run.smithi152.stdout: URL: https://smithi152.front.sepia.ceph.com:8443/ 2024-04-02T04:50:07.830 INFO:teuthology.orchestra.run.smithi152.stdout: User: admin 2024-04-02T04:50:07.830 INFO:teuthology.orchestra.run.smithi152.stdout: Password: 9kcvtrultq 2024-04-02T04:50:07.830 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:50:07.830 INFO:teuthology.orchestra.run.smithi152.stdout:Saving cluster configuration to /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/config directory 2024-04-02T04:50:07.830 INFO:teuthology.orchestra.run.smithi152.stdout:Enabling autotune for osd_memory_target 2024-04-02T04:50:07.952 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:07 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/121925111' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-02T04:50:09.031 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.246+0000 7f3c5fa14700 1 Processor -- start 2024-04-02T04:50:09.031 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.246+0000 7f3c5fa14700 1 -- start start 2024-04-02T04:50:09.031 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.246+0000 7f3c5fa14700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3c58076ce0 0x7f3c581054c0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:09.031 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.246+0000 7f3c5fa14700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c58105a00 con 0x7f3c58076ce0 2024-04-02T04:50:09.031 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.247+0000 7f3c5d7b0700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3c58076ce0 0x7f3c581054c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:09.031 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.247+0000 7f3c5d7b0700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3c58076ce0 0x7f3c581054c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:34574/0 (socket says 172.21.15.152:34574) 2024-04-02T04:50:09.031 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.247+0000 7f3c5d7b0700 1 -- 172.21.15.152:0/569911075 learned_addr learned my addr 172.21.15.152:0/569911075 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:50:09.031 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.247+0000 7f3c5d7b0700 1 -- 172.21.15.152:0/569911075 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3c58105b40 con 0x7f3c58076ce0 2024-04-02T04:50:09.031 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.247+0000 7f3c5d7b0700 1 --2- 172.21.15.152:0/569911075 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3c58076ce0 0x7f3c581054c0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f3c480098e0 tx=0x7f3c48009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=fcf040dd0ea7ee1e server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:09.031 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.247+0000 7f3c4ffff700 1 -- 172.21.15.152:0/569911075 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f3c4800b860 con 0x7f3c58076ce0 2024-04-02T04:50:09.031 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.247+0000 7f3c4ffff700 1 -- 172.21.15.152:0/569911075 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f3c4800be90 con 0x7f3c58076ce0 2024-04-02T04:50:09.031 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.247+0000 7f3c4ffff700 1 -- 172.21.15.152:0/569911075 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f3c480103c0 con 0x7f3c58076ce0 2024-04-02T04:50:09.031 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.248+0000 7f3c5fa14700 1 -- 172.21.15.152:0/569911075 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3c58076ce0 msgr2=0x7f3c581054c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:09.032 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.248+0000 7f3c5fa14700 1 --2- 172.21.15.152:0/569911075 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3c58076ce0 0x7f3c581054c0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7f3c480098e0 tx=0x7f3c48009250 comp rx=0 tx=0).stop 2024-04-02T04:50:09.032 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.248+0000 7f3c5fa14700 1 -- 172.21.15.152:0/569911075 shutdown_connections 2024-04-02T04:50:09.032 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.248+0000 7f3c5fa14700 1 --2- 172.21.15.152:0/569911075 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3c58076ce0 0x7f3c581054c0 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:09.032 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.248+0000 7f3c5fa14700 1 -- 172.21.15.152:0/569911075 >> 172.21.15.152:0/569911075 conn(0x7f3c580fcf70 msgr2=0x7f3c580ff380 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:09.032 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.248+0000 7f3c5fa14700 1 -- 172.21.15.152:0/569911075 shutdown_connections 2024-04-02T04:50:09.032 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.248+0000 7f3c5fa14700 1 -- 172.21.15.152:0/569911075 wait complete. 2024-04-02T04:50:09.032 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.248+0000 7f3c5fa14700 1 Processor -- start 2024-04-02T04:50:09.032 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.248+0000 7f3c5fa14700 1 -- start start 2024-04-02T04:50:09.032 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.249+0000 7f3c5fa14700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3c58076ce0 0x7f3c581116e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:09.032 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.249+0000 7f3c5fa14700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3c58111c20 con 0x7f3c58076ce0 2024-04-02T04:50:09.032 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.249+0000 7f3c5d7b0700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3c58076ce0 0x7f3c581116e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:09.032 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.249+0000 7f3c5d7b0700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3c58076ce0 0x7f3c581116e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:34578/0 (socket says 172.21.15.152:34578) 2024-04-02T04:50:09.032 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.249+0000 7f3c5d7b0700 1 -- 172.21.15.152:0/388859433 learned_addr learned my addr 172.21.15.152:0/388859433 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:50:09.032 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.249+0000 7f3c5d7b0700 1 -- 172.21.15.152:0/388859433 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3c480094f0 con 0x7f3c58076ce0 2024-04-02T04:50:09.033 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.249+0000 7f3c5d7b0700 1 --2- 172.21.15.152:0/388859433 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3c58076ce0 0x7f3c581116e0 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f3c480094c0 tx=0x7f3c48009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:09.033 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.249+0000 7f3c4dffb700 1 -- 172.21.15.152:0/388859433 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f3c48010890 con 0x7f3c58076ce0 2024-04-02T04:50:09.033 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.249+0000 7f3c5fa14700 1 -- 172.21.15.152:0/388859433 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3c58111e20 con 0x7f3c58076ce0 2024-04-02T04:50:09.033 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.249+0000 7f3c4dffb700 1 -- 172.21.15.152:0/388859433 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f3c48010ec0 con 0x7f3c58076ce0 2024-04-02T04:50:09.033 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.249+0000 7f3c4dffb700 1 -- 172.21.15.152:0/388859433 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f3c480243f0 con 0x7f3c58076ce0 2024-04-02T04:50:09.033 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.249+0000 7f3c5fa14700 1 -- 172.21.15.152:0/388859433 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3c58112990 con 0x7f3c58076ce0 2024-04-02T04:50:09.033 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.250+0000 7f3c4dffb700 1 -- 172.21.15.152:0/388859433 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 12) v1 ==== 43381+0+0 (secure 0 0 0) 0x7f3c4802d3f0 con 0x7f3c58076ce0 2024-04-02T04:50:09.033 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.250+0000 7f3c4dffb700 1 --2- 172.21.15.152:0/388859433 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f3c44036830 0x7f3c44038ce0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:09.033 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.250+0000 7f3c5fa14700 1 -- 172.21.15.152:0/388859433 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3c581122b0 con 0x7f3c58076ce0 2024-04-02T04:50:09.033 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.250+0000 7f3c4dffb700 1 -- 172.21.15.152:0/388859433 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1151+0+0 (secure 0 0 0) 0x7f3c4804a520 con 0x7f3c58076ce0 2024-04-02T04:50:09.033 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.251+0000 7f3c5cfaf700 1 --2- 172.21.15.152:0/388859433 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f3c44036830 0x7f3c44038ce0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:09.033 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.251+0000 7f3c5cfaf700 1 --2- 172.21.15.152:0/388859433 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f3c44036830 0x7f3c44038ce0 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f3c54009990 tx=0x7f3c54006e40 comp rx=0 tx=0).ready entity=mgr.14160 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:09.033 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.255+0000 7f3c4dffb700 1 -- 172.21.15.152:0/388859433 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f3c48018ce0 con 0x7f3c58076ce0 2024-04-02T04:50:09.033 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.442+0000 7f3c5fa14700 1 -- 172.21.15.152:0/388859433 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command([{prefix=config set, name=osd_memory_target_autotune}] v 0) v1 -- 0x7f3c5805f5f0 con 0x7f3c58076ce0 2024-04-02T04:50:09.034 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.442+0000 7f3c4dffb700 1 -- 172.21.15.152:0/388859433 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{prefix=config set, name=osd_memory_target_autotune}]=0 v8)=0 v8) v1 ==== 127+0+0 (secure 0 0 0) 0x7f3c48018b00 con 0x7f3c58076ce0 2024-04-02T04:50:09.034 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.444+0000 7f3c5fa14700 1 -- 172.21.15.152:0/388859433 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f3c44036830 msgr2=0x7f3c44038ce0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:09.034 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.444+0000 7f3c5fa14700 1 --2- 172.21.15.152:0/388859433 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f3c44036830 0x7f3c44038ce0 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 crypto rx=0x7f3c54009990 tx=0x7f3c54006e40 comp rx=0 tx=0).stop 2024-04-02T04:50:09.034 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.445+0000 7f3c5fa14700 1 -- 172.21.15.152:0/388859433 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3c58076ce0 msgr2=0x7f3c581116e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:09.034 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.445+0000 7f3c5fa14700 1 --2- 172.21.15.152:0/388859433 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3c58076ce0 0x7f3c581116e0 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f3c480094c0 tx=0x7f3c48009250 comp rx=0 tx=0).stop 2024-04-02T04:50:09.034 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.445+0000 7f3c5fa14700 1 -- 172.21.15.152:0/388859433 shutdown_connections 2024-04-02T04:50:09.034 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.445+0000 7f3c5fa14700 1 --2- 172.21.15.152:0/388859433 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f3c44036830 0x7f3c44038ce0 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:09.034 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.445+0000 7f3c5fa14700 1 --2- 172.21.15.152:0/388859433 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3c58076ce0 0x7f3c581116e0 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:09.034 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.445+0000 7f3c5fa14700 1 -- 172.21.15.152:0/388859433 >> 172.21.15.152:0/388859433 conn(0x7f3c580fcf70 msgr2=0x7f3c580fda80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:09.034 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.445+0000 7f3c5fa14700 1 -- 172.21.15.152:0/388859433 shutdown_connections 2024-04-02T04:50:09.034 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:08.445+0000 7f3c5fa14700 1 -- 172.21.15.152:0/388859433 wait complete. 2024-04-02T04:50:10.006 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:09 smithi152 ceph-mon[29073]: mgrmap e13: smithi152.jivdcl(active, since 6s) 2024-04-02T04:50:10.355 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.452+0000 7f2f9ec3d700 1 Processor -- start 2024-04-02T04:50:10.355 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.452+0000 7f2f9ec3d700 1 -- start start 2024-04-02T04:50:10.355 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.452+0000 7f2f9ec3d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2f98102920 0x7f2f98102d30 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:10.355 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.452+0000 7f2f9ec3d700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2f98103270 con 0x7f2f98102920 2024-04-02T04:50:10.355 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.452+0000 7f2f9c9d9700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2f98102920 0x7f2f98102d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:10.355 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.452+0000 7f2f9c9d9700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2f98102920 0x7f2f98102d30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:34586/0 (socket says 172.21.15.152:34586) 2024-04-02T04:50:10.355 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.452+0000 7f2f9c9d9700 1 -- 172.21.15.152:0/2883497628 learned_addr learned my addr 172.21.15.152:0/2883497628 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:50:10.355 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.452+0000 7f2f9c9d9700 1 -- 172.21.15.152:0/2883497628 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2f981033b0 con 0x7f2f98102920 2024-04-02T04:50:10.355 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.452+0000 7f2f9c9d9700 1 --2- 172.21.15.152:0/2883497628 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2f98102920 0x7f2f98102d30 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f2f8c0098e0 tx=0x7f2f8c009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=e44ca1b4b4e9d7fd server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:10.355 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.453+0000 7f2f977fe700 1 -- 172.21.15.152:0/2883497628 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f2f8c00b860 con 0x7f2f98102920 2024-04-02T04:50:10.355 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.453+0000 7f2f977fe700 1 -- 172.21.15.152:0/2883497628 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f2f8c00be90 con 0x7f2f98102920 2024-04-02T04:50:10.356 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.453+0000 7f2f977fe700 1 -- 172.21.15.152:0/2883497628 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f2f8c0103c0 con 0x7f2f98102920 2024-04-02T04:50:10.356 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.453+0000 7f2f9ec3d700 1 -- 172.21.15.152:0/2883497628 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2f98102920 msgr2=0x7f2f98102d30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:10.356 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.453+0000 7f2f9ec3d700 1 --2- 172.21.15.152:0/2883497628 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2f98102920 0x7f2f98102d30 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f2f8c0098e0 tx=0x7f2f8c009250 comp rx=0 tx=0).stop 2024-04-02T04:50:10.356 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.453+0000 7f2f9ec3d700 1 -- 172.21.15.152:0/2883497628 shutdown_connections 2024-04-02T04:50:10.356 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.453+0000 7f2f9ec3d700 1 --2- 172.21.15.152:0/2883497628 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2f98102920 0x7f2f98102d30 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:10.356 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.453+0000 7f2f9ec3d700 1 -- 172.21.15.152:0/2883497628 >> 172.21.15.152:0/2883497628 conn(0x7f2f980fe0a0 msgr2=0x7f2f981004b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:10.356 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.453+0000 7f2f9ec3d700 1 -- 172.21.15.152:0/2883497628 shutdown_connections 2024-04-02T04:50:10.356 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.453+0000 7f2f9ec3d700 1 -- 172.21.15.152:0/2883497628 wait complete. 2024-04-02T04:50:10.356 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.454+0000 7f2f9ec3d700 1 Processor -- start 2024-04-02T04:50:10.356 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.454+0000 7f2f9ec3d700 1 -- start start 2024-04-02T04:50:10.356 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.454+0000 7f2f9ec3d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2f98102920 0x7f2f98113900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:10.356 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.454+0000 7f2f9ec3d700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2f98113e40 con 0x7f2f98102920 2024-04-02T04:50:10.356 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.454+0000 7f2f9c9d9700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2f98102920 0x7f2f98113900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:10.356 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.455+0000 7f2f9c9d9700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2f98102920 0x7f2f98113900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:34588/0 (socket says 172.21.15.152:34588) 2024-04-02T04:50:10.357 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.455+0000 7f2f9c9d9700 1 -- 172.21.15.152:0/2022705229 learned_addr learned my addr 172.21.15.152:0/2022705229 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:50:10.357 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.455+0000 7f2f9c9d9700 1 -- 172.21.15.152:0/2022705229 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2f8c0094f0 con 0x7f2f98102920 2024-04-02T04:50:10.357 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.455+0000 7f2f9c9d9700 1 --2- 172.21.15.152:0/2022705229 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2f98102920 0x7f2f98113900 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f2f8c011680 tx=0x7f2f8c009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:10.357 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.455+0000 7f2f957fa700 1 -- 172.21.15.152:0/2022705229 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f2f8c010890 con 0x7f2f98102920 2024-04-02T04:50:10.357 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.455+0000 7f2f957fa700 1 -- 172.21.15.152:0/2022705229 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f2f8c010ec0 con 0x7f2f98102920 2024-04-02T04:50:10.357 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.455+0000 7f2f9ec3d700 1 -- 172.21.15.152:0/2022705229 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2f98114040 con 0x7f2f98102920 2024-04-02T04:50:10.357 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.455+0000 7f2f957fa700 1 -- 172.21.15.152:0/2022705229 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f2f8c0243f0 con 0x7f2f98102920 2024-04-02T04:50:10.357 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.455+0000 7f2f9ec3d700 1 -- 172.21.15.152:0/2022705229 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2f98114bb0 con 0x7f2f98102920 2024-04-02T04:50:10.357 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.456+0000 7f2f957fa700 1 -- 172.21.15.152:0/2022705229 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 13) v1 ==== 43425+0+0 (secure 0 0 0) 0x7f2f8c02d3f0 con 0x7f2f98102920 2024-04-02T04:50:10.357 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.456+0000 7f2f9ec3d700 1 -- 172.21.15.152:0/2022705229 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2f9805f5f0 con 0x7f2f98102920 2024-04-02T04:50:10.357 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.456+0000 7f2f957fa700 1 --2- 172.21.15.152:0/2022705229 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f2f88036880 0x7f2f88038d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:10.357 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.456+0000 7f2f957fa700 1 -- 172.21.15.152:0/2022705229 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1151+0+0 (secure 0 0 0) 0x7f2f8c04a450 con 0x7f2f98102920 2024-04-02T04:50:10.358 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.456+0000 7f2f97fff700 1 --2- 172.21.15.152:0/2022705229 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f2f88036880 0x7f2f88038d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:10.358 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.457+0000 7f2f97fff700 1 --2- 172.21.15.152:0/2022705229 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f2f88036880 0x7f2f88038d30 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f2f84009990 tx=0x7f2f84006e40 comp rx=0 tx=0).ready entity=mgr.14160 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:10.358 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.461+0000 7f2f957fa700 1 -- 172.21.15.152:0/2022705229 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f2f8c006d70 con 0x7f2f98102920 2024-04-02T04:50:10.358 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.750+0000 7f2f9ec3d700 1 -- 172.21.15.152:0/2022705229 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command([{prefix=config-key set, key=mgr/dashboard/cluster/status}] v 0) v1 -- 0x7f2f9804bed0 con 0x7f2f98102920 2024-04-02T04:50:10.358 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.755+0000 7f2f957fa700 1 -- 172.21.15.152:0/2022705229 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{prefix=config-key set, key=mgr/dashboard/cluster/status}]=0 set mgr/dashboard/cluster/status v33)=0 set mgr/dashboard/cluster/status v33) v1 ==== 153+0+0 (secure 0 0 0) 0x7f2f8c02c350 con 0x7f2f98102920 2024-04-02T04:50:10.358 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.757+0000 7f2f9ec3d700 1 -- 172.21.15.152:0/2022705229 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f2f88036880 msgr2=0x7f2f88038d30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:10.358 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.757+0000 7f2f9ec3d700 1 --2- 172.21.15.152:0/2022705229 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f2f88036880 0x7f2f88038d30 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 crypto rx=0x7f2f84009990 tx=0x7f2f84006e40 comp rx=0 tx=0).stop 2024-04-02T04:50:10.358 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.757+0000 7f2f9ec3d700 1 -- 172.21.15.152:0/2022705229 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2f98102920 msgr2=0x7f2f98113900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:10.358 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.757+0000 7f2f9ec3d700 1 --2- 172.21.15.152:0/2022705229 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2f98102920 0x7f2f98113900 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f2f8c011680 tx=0x7f2f8c009250 comp rx=0 tx=0).stop 2024-04-02T04:50:10.358 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.757+0000 7f2f9ec3d700 1 -- 172.21.15.152:0/2022705229 shutdown_connections 2024-04-02T04:50:10.358 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.757+0000 7f2f9ec3d700 1 --2- 172.21.15.152:0/2022705229 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f2f88036880 0x7f2f88038d30 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:10.358 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.757+0000 7f2f9ec3d700 1 --2- 172.21.15.152:0/2022705229 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2f98102920 0x7f2f98113900 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:10.358 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.757+0000 7f2f9ec3d700 1 -- 172.21.15.152:0/2022705229 >> 172.21.15.152:0/2022705229 conn(0x7f2f980fe0a0 msgr2=0x7f2f980fec90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:10.359 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.757+0000 7f2f9ec3d700 1 -- 172.21.15.152:0/2022705229 shutdown_connections 2024-04-02T04:50:10.359 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr 2024-04-02T04:50:09.757+0000 7f2f9ec3d700 1 -- 172.21.15.152:0/2022705229 wait complete. 2024-04-02T04:50:10.359 INFO:teuthology.orchestra.run.smithi152.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-04-02T04:50:10.359 INFO:teuthology.orchestra.run.smithi152.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-04-02T04:50:10.359 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:50:10.359 INFO:teuthology.orchestra.run.smithi152.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-02T04:50:10.359 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:50:10.359 INFO:teuthology.orchestra.run.smithi152.stdout:Or, if you are only running a single cluster on this host: 2024-04-02T04:50:10.359 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:50:10.359 INFO:teuthology.orchestra.run.smithi152.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-04-02T04:50:10.359 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:50:10.359 INFO:teuthology.orchestra.run.smithi152.stdout:Please consider enabling telemetry to help improve Ceph: 2024-04-02T04:50:10.359 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:50:10.359 INFO:teuthology.orchestra.run.smithi152.stdout: ceph telemetry on 2024-04-02T04:50:10.359 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:50:10.359 INFO:teuthology.orchestra.run.smithi152.stdout:For more information see: 2024-04-02T04:50:10.359 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:50:10.360 INFO:teuthology.orchestra.run.smithi152.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-02T04:50:10.360 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:50:10.360 INFO:teuthology.orchestra.run.smithi152.stdout:Bootstrap complete. 2024-04-02T04:50:10.389 INFO:tasks.cephadm:Fetching config... 2024-04-02T04:50:10.390 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-02T04:50:10.390 DEBUG:teuthology.orchestra.run.smithi152:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-02T04:50:10.405 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-02T04:50:10.405 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-02T04:50:10.405 DEBUG:teuthology.orchestra.run.smithi152:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-02T04:50:10.461 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-02T04:50:10.461 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-02T04:50:10.461 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd if=/var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/keyring of=/dev/stdout 2024-04-02T04:50:10.528 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-02T04:50:10.529 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-02T04:50:10.529 DEBUG:teuthology.orchestra.run.smithi152:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-02T04:50:10.585 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-02T04:50:10.585 DEBUG:teuthology.orchestra.run.smithi152:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDqUGpvT48ABXhnBd4AJXZxnZ2UI42+coiT5cnqfJddP/Ccx1tK0AJA0Iep7wybj+Fi/JmuMw0nSP2YDrfrb1tOOhQzr1Z1BIoYMS5LJlMw6Xfj5wXauoAydDAc5duMr0vTy49iZmHzqvw4scsLw8xsq8Z8TimRtMf4wUaGqiS9bbQ+mq4L9TBFjtznBdMaJ/34FAINzZpatWnndVUy59Q0onixMKkalhNfNEqnR0djmp137y1xQCxZ4M+EJzTaz912zQtNO1Xi55bzWmupb5xk34JeaoSU5KwjUx9k5IgO6qlg9d8GujXFQ7U9HwyV9ShLEMiLTSaF2jJJmV3+ClX+R0V0hG5XlZiBrOxAW7idolGX3qxAk8S8kw9RLEwmd6tylzB5eGb6Z80UUlOfkYRBhdk60AJDqc8ra0sF0aK++c57DR78f4xJ4VNjGTsbk2mTIFZEtM6Vmu3NDqp9qmyHSMAzfT8s+HmtX+IwyN3hXyI+ajNeTi50dHi6seIV/us= ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-02T04:50:10.664 INFO:teuthology.orchestra.run.smithi152.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDqUGpvT48ABXhnBd4AJXZxnZ2UI42+coiT5cnqfJddP/Ccx1tK0AJA0Iep7wybj+Fi/JmuMw0nSP2YDrfrb1tOOhQzr1Z1BIoYMS5LJlMw6Xfj5wXauoAydDAc5duMr0vTy49iZmHzqvw4scsLw8xsq8Z8TimRtMf4wUaGqiS9bbQ+mq4L9TBFjtznBdMaJ/34FAINzZpatWnndVUy59Q0onixMKkalhNfNEqnR0djmp137y1xQCxZ4M+EJzTaz912zQtNO1Xi55bzWmupb5xk34JeaoSU5KwjUx9k5IgO6qlg9d8GujXFQ7U9HwyV9ShLEMiLTSaF2jJJmV3+ClX+R0V0hG5XlZiBrOxAW7idolGX3qxAk8S8kw9RLEwmd6tylzB5eGb6Z80UUlOfkYRBhdk60AJDqc8ra0sF0aK++c57DR78f4xJ4VNjGTsbk2mTIFZEtM6Vmu3NDqp9qmyHSMAzfT8s+HmtX+IwyN3hXyI+ajNeTi50dHi6seIV/us= ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4 2024-04-02T04:50:10.680 DEBUG:teuthology.orchestra.run.smithi154:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDqUGpvT48ABXhnBd4AJXZxnZ2UI42+coiT5cnqfJddP/Ccx1tK0AJA0Iep7wybj+Fi/JmuMw0nSP2YDrfrb1tOOhQzr1Z1BIoYMS5LJlMw6Xfj5wXauoAydDAc5duMr0vTy49iZmHzqvw4scsLw8xsq8Z8TimRtMf4wUaGqiS9bbQ+mq4L9TBFjtznBdMaJ/34FAINzZpatWnndVUy59Q0onixMKkalhNfNEqnR0djmp137y1xQCxZ4M+EJzTaz912zQtNO1Xi55bzWmupb5xk34JeaoSU5KwjUx9k5IgO6qlg9d8GujXFQ7U9HwyV9ShLEMiLTSaF2jJJmV3+ClX+R0V0hG5XlZiBrOxAW7idolGX3qxAk8S8kw9RLEwmd6tylzB5eGb6Z80UUlOfkYRBhdk60AJDqc8ra0sF0aK++c57DR78f4xJ4VNjGTsbk2mTIFZEtM6Vmu3NDqp9qmyHSMAzfT8s+HmtX+IwyN3hXyI+ajNeTi50dHi6seIV/us= ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-02T04:50:10.728 INFO:teuthology.orchestra.run.smithi154.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDqUGpvT48ABXhnBd4AJXZxnZ2UI42+coiT5cnqfJddP/Ccx1tK0AJA0Iep7wybj+Fi/JmuMw0nSP2YDrfrb1tOOhQzr1Z1BIoYMS5LJlMw6Xfj5wXauoAydDAc5duMr0vTy49iZmHzqvw4scsLw8xsq8Z8TimRtMf4wUaGqiS9bbQ+mq4L9TBFjtznBdMaJ/34FAINzZpatWnndVUy59Q0onixMKkalhNfNEqnR0djmp137y1xQCxZ4M+EJzTaz912zQtNO1Xi55bzWmupb5xk34JeaoSU5KwjUx9k5IgO6qlg9d8GujXFQ7U9HwyV9ShLEMiLTSaF2jJJmV3+ClX+R0V0hG5XlZiBrOxAW7idolGX3qxAk8S8kw9RLEwmd6tylzB5eGb6Z80UUlOfkYRBhdk60AJDqc8ra0sF0aK++c57DR78f4xJ4VNjGTsbk2mTIFZEtM6Vmu3NDqp9qmyHSMAzfT8s+HmtX+IwyN3hXyI+ajNeTi50dHi6seIV/us= ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4 2024-04-02T04:50:10.743 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-02T04:50:11.003 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:50:11.006 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:10 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/2022705229' entity='client.admin' 2024-04-02T04:50:13.186 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.181+0000 7f03a7eaa700 1 -- 172.21.15.152:0/3044798817 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f03a00fca40 msgr2=0x7f03a00fce50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:13.186 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.181+0000 7f03a7eaa700 1 --2- 172.21.15.152:0/3044798817 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f03a00fca40 0x7f03a00fce50 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7f039c0048c0 tx=0x7f039c009e30 comp rx=0 tx=0).stop 2024-04-02T04:50:13.186 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.181+0000 7f03a7eaa700 1 -- 172.21.15.152:0/3044798817 shutdown_connections 2024-04-02T04:50:13.186 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.181+0000 7f03a7eaa700 1 --2- 172.21.15.152:0/3044798817 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f03a00fca40 0x7f03a00fce50 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:13.186 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.181+0000 7f03a7eaa700 1 -- 172.21.15.152:0/3044798817 >> 172.21.15.152:0/3044798817 conn(0x7f03a00f81c0 msgr2=0x7f03a00fa5d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:13.186 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.181+0000 7f03a7eaa700 1 -- 172.21.15.152:0/3044798817 shutdown_connections 2024-04-02T04:50:13.186 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.181+0000 7f03a7eaa700 1 -- 172.21.15.152:0/3044798817 wait complete. 2024-04-02T04:50:13.186 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.181+0000 7f03a7eaa700 1 Processor -- start 2024-04-02T04:50:13.186 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.181+0000 7f03a7eaa700 1 -- start start 2024-04-02T04:50:13.187 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.181+0000 7f03a7eaa700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f03a00fca40 0x7f03a010d0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:13.187 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.181+0000 7f03a7eaa700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f03a010d600 con 0x7f03a00fca40 2024-04-02T04:50:13.187 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.182+0000 7f03a5c46700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f03a00fca40 0x7f03a010d0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:13.187 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.182+0000 7f03a5c46700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f03a00fca40 0x7f03a010d0c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:38206/0 (socket says 172.21.15.152:38206) 2024-04-02T04:50:13.187 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.182+0000 7f03a5c46700 1 -- 172.21.15.152:0/3874621395 learned_addr learned my addr 172.21.15.152:0/3874621395 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:50:13.187 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.182+0000 7f03a5c46700 1 -- 172.21.15.152:0/3874621395 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f039c005040 con 0x7f03a00fca40 2024-04-02T04:50:13.187 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.182+0000 7f03a5c46700 1 --2- 172.21.15.152:0/3874621395 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f03a00fca40 0x7f03a010d0c0 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f039c0096b0 tx=0x7f039c009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:13.187 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.182+0000 7f03967fc700 1 -- 172.21.15.152:0/3874621395 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f039c00bed0 con 0x7f03a00fca40 2024-04-02T04:50:13.187 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.182+0000 7f03a7eaa700 1 -- 172.21.15.152:0/3874621395 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f03a010d800 con 0x7f03a00fca40 2024-04-02T04:50:13.187 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.182+0000 7f03967fc700 1 -- 172.21.15.152:0/3874621395 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f039c00b5c0 con 0x7f03a00fca40 2024-04-02T04:50:13.187 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.182+0000 7f03967fc700 1 -- 172.21.15.152:0/3874621395 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f039c01c5b0 con 0x7f03a00fca40 2024-04-02T04:50:13.187 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.182+0000 7f03a7eaa700 1 -- 172.21.15.152:0/3874621395 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f03a010e370 con 0x7f03a00fca40 2024-04-02T04:50:13.188 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.183+0000 7f03967fc700 1 -- 172.21.15.152:0/3874621395 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 13) v1 ==== 43425+0+0 (secure 0 0 0) 0x7f039c013420 con 0x7f03a00fca40 2024-04-02T04:50:13.188 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.183+0000 7f03a7eaa700 1 -- 172.21.15.152:0/3874621395 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f03a0102f50 con 0x7f03a00fca40 2024-04-02T04:50:13.188 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.183+0000 7f03967fc700 1 --2- 172.21.15.152:0/3874621395 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f038c0368d0 0x7f038c038d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:13.188 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.183+0000 7f03967fc700 1 -- 172.21.15.152:0/3874621395 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1151+0+0 (secure 0 0 0) 0x7f039c049a00 con 0x7f03a00fca40 2024-04-02T04:50:13.188 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.183+0000 7f03a5445700 1 --2- 172.21.15.152:0/3874621395 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f038c0368d0 0x7f038c038d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:13.188 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.184+0000 7f03a5445700 1 --2- 172.21.15.152:0/3874621395 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f038c0368d0 0x7f038c038d80 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f0390009990 tx=0x7f0390006e40 comp rx=0 tx=0).ready entity=mgr.14160 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:13.191 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.188+0000 7f03967fc700 1 -- 172.21.15.152:0/3874621395 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f039c004d30 con 0x7f03a00fca40 2024-04-02T04:50:13.379 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.375+0000 7f03a7eaa700 1 -- 172.21.15.152:0/3874621395 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command([{prefix=config set, name=mgr/cephadm/allow_ptrace}] v 0) v1 -- 0x7f03a005f5f0 con 0x7f03a00fca40 2024-04-02T04:50:13.382 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.379+0000 7f03967fc700 1 -- 172.21.15.152:0/3874621395 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/cephadm/allow_ptrace}]=0 v9)=0 v9) v1 ==== 125+0+0 (secure 0 0 0) 0x7f039c004d30 con 0x7f03a00fca40 2024-04-02T04:50:13.384 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.381+0000 7f03a7eaa700 1 -- 172.21.15.152:0/3874621395 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f038c0368d0 msgr2=0x7f038c038d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:13.384 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.381+0000 7f03a7eaa700 1 --2- 172.21.15.152:0/3874621395 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f038c0368d0 0x7f038c038d80 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 crypto rx=0x7f0390009990 tx=0x7f0390006e40 comp rx=0 tx=0).stop 2024-04-02T04:50:13.384 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.381+0000 7f03a7eaa700 1 -- 172.21.15.152:0/3874621395 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f03a00fca40 msgr2=0x7f03a010d0c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:13.385 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.381+0000 7f03a7eaa700 1 --2- 172.21.15.152:0/3874621395 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f03a00fca40 0x7f03a010d0c0 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7f039c0096b0 tx=0x7f039c009e30 comp rx=0 tx=0).stop 2024-04-02T04:50:13.385 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.381+0000 7f03a7eaa700 1 -- 172.21.15.152:0/3874621395 shutdown_connections 2024-04-02T04:50:13.385 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.381+0000 7f03a7eaa700 1 --2- 172.21.15.152:0/3874621395 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f038c0368d0 0x7f038c038d80 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:13.385 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.381+0000 7f03a7eaa700 1 --2- 172.21.15.152:0/3874621395 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f03a00fca40 0x7f03a010d0c0 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:13.385 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.381+0000 7f03a7eaa700 1 -- 172.21.15.152:0/3874621395 >> 172.21.15.152:0/3874621395 conn(0x7f03a00f81c0 msgr2=0x7f03a0103b80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:13.385 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.381+0000 7f03a7eaa700 1 -- 172.21.15.152:0/3874621395 shutdown_connections 2024-04-02T04:50:13.385 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:13.381+0000 7f03a7eaa700 1 -- 172.21.15.152:0/3874621395 wait complete. 2024-04-02T04:50:14.143 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-02T04:50:14.143 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-02T04:50:14.421 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:50:14.710 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:14 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/3874621395' entity='client.admin' 2024-04-02T04:50:16.384 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.381+0000 7f3131c36700 1 -- 172.21.15.152:0/265134662 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f312c0fca50 msgr2=0x7f312c0fce60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:16.384 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.381+0000 7f3131c36700 1 --2- 172.21.15.152:0/265134662 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f312c0fca50 0x7f312c0fce60 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f311c0048c0 tx=0x7f311c009e30 comp rx=0 tx=0).stop 2024-04-02T04:50:16.385 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.381+0000 7f3131c36700 1 -- 172.21.15.152:0/265134662 shutdown_connections 2024-04-02T04:50:16.385 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.381+0000 7f3131c36700 1 --2- 172.21.15.152:0/265134662 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f312c0fca50 0x7f312c0fce60 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:16.385 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.381+0000 7f3131c36700 1 -- 172.21.15.152:0/265134662 >> 172.21.15.152:0/265134662 conn(0x7f312c0f8190 msgr2=0x7f312c0fa5e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:16.385 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.381+0000 7f3131c36700 1 -- 172.21.15.152:0/265134662 shutdown_connections 2024-04-02T04:50:16.385 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.381+0000 7f3131c36700 1 -- 172.21.15.152:0/265134662 wait complete. 2024-04-02T04:50:16.386 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.382+0000 7f3131c36700 1 Processor -- start 2024-04-02T04:50:16.386 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.382+0000 7f3131c36700 1 -- start start 2024-04-02T04:50:16.386 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.382+0000 7f3131c36700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f312c0fca50 0x7f312c10d0a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:16.386 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.382+0000 7f3131c36700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f312c10d5e0 con 0x7f312c0fca50 2024-04-02T04:50:16.386 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.382+0000 7f312b7fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f312c0fca50 0x7f312c10d0a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:16.387 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.382+0000 7f312b7fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f312c0fca50 0x7f312c10d0a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:38224/0 (socket says 172.21.15.152:38224) 2024-04-02T04:50:16.387 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.382+0000 7f312b7fe700 1 -- 172.21.15.152:0/2516741971 learned_addr learned my addr 172.21.15.152:0/2516741971 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:50:16.387 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.382+0000 7f312b7fe700 1 -- 172.21.15.152:0/2516741971 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f311c005040 con 0x7f312c0fca50 2024-04-02T04:50:16.387 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.383+0000 7f312b7fe700 1 --2- 172.21.15.152:0/2516741971 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f312c0fca50 0x7f312c10d0a0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f311c0096b0 tx=0x7f311c009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:16.387 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.383+0000 7f3113fff700 1 -- 172.21.15.152:0/2516741971 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f311c00bed0 con 0x7f312c0fca50 2024-04-02T04:50:16.387 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.383+0000 7f3131c36700 1 -- 172.21.15.152:0/2516741971 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f312c10d7e0 con 0x7f312c0fca50 2024-04-02T04:50:16.387 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.383+0000 7f3113fff700 1 -- 172.21.15.152:0/2516741971 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f311c00b5c0 con 0x7f312c0fca50 2024-04-02T04:50:16.387 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.383+0000 7f3131c36700 1 -- 172.21.15.152:0/2516741971 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f312c10e350 con 0x7f312c0fca50 2024-04-02T04:50:16.387 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.383+0000 7f3113fff700 1 -- 172.21.15.152:0/2516741971 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f311c01c5b0 con 0x7f312c0fca50 2024-04-02T04:50:16.388 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.384+0000 7f3113fff700 1 -- 172.21.15.152:0/2516741971 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 13) v1 ==== 43425+0+0 (secure 0 0 0) 0x7f311c01c710 con 0x7f312c0fca50 2024-04-02T04:50:16.388 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.384+0000 7f3131c36700 1 -- 172.21.15.152:0/2516741971 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f312c102f60 con 0x7f312c0fca50 2024-04-02T04:50:16.388 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.384+0000 7f3113fff700 1 --2- 172.21.15.152:0/2516741971 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f31140368d0 0x7f3114038d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:16.389 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.384+0000 7f3113fff700 1 -- 172.21.15.152:0/2516741971 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1151+0+0 (secure 0 0 0) 0x7f311c0499a0 con 0x7f312c0fca50 2024-04-02T04:50:16.389 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.384+0000 7f312affd700 1 --2- 172.21.15.152:0/2516741971 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f31140368d0 0x7f3114038d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:16.389 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.385+0000 7f312affd700 1 --2- 172.21.15.152:0/2516741971 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f31140368d0 0x7f3114038d80 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f3120009990 tx=0x7f3120006e40 comp rx=0 tx=0).ready entity=mgr.14160 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:16.392 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.389+0000 7f3113fff700 1 -- 172.21.15.152:0/2516741971 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f311c025400 con 0x7f312c0fca50 2024-04-02T04:50:16.598 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.595+0000 7f3131c36700 1 -- 172.21.15.152:0/2516741971 --> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] -- mgr_command(tid 0: {"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}) v1 -- 0x7f312c10e590 con 0x7f31140368d0 2024-04-02T04:50:16.601 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.598+0000 7f3113fff700 1 -- 172.21.15.152:0/2516741971 <== mgr.14160 v2:172.21.15.152:6800/2497421829 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+0 (secure 0 0 0) 0x7f312c10e590 con 0x7f31140368d0 2024-04-02T04:50:16.604 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.600+0000 7f3131c36700 1 -- 172.21.15.152:0/2516741971 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f31140368d0 msgr2=0x7f3114038d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:16.604 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.600+0000 7f3131c36700 1 --2- 172.21.15.152:0/2516741971 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f31140368d0 0x7f3114038d80 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 crypto rx=0x7f3120009990 tx=0x7f3120006e40 comp rx=0 tx=0).stop 2024-04-02T04:50:16.604 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.600+0000 7f3131c36700 1 -- 172.21.15.152:0/2516741971 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f312c0fca50 msgr2=0x7f312c10d0a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:16.604 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.600+0000 7f3131c36700 1 --2- 172.21.15.152:0/2516741971 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f312c0fca50 0x7f312c10d0a0 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f311c0096b0 tx=0x7f311c009e30 comp rx=0 tx=0).stop 2024-04-02T04:50:16.604 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.600+0000 7f3131c36700 1 -- 172.21.15.152:0/2516741971 shutdown_connections 2024-04-02T04:50:16.604 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.600+0000 7f3131c36700 1 --2- 172.21.15.152:0/2516741971 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f31140368d0 0x7f3114038d80 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:16.604 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.600+0000 7f3131c36700 1 --2- 172.21.15.152:0/2516741971 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f312c0fca50 0x7f312c10d0a0 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:16.604 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.600+0000 7f3131c36700 1 -- 172.21.15.152:0/2516741971 >> 172.21.15.152:0/2516741971 conn(0x7f312c0f8190 msgr2=0x7f312c103b90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:16.604 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.600+0000 7f3131c36700 1 -- 172.21.15.152:0/2516741971 shutdown_connections 2024-04-02T04:50:16.604 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:16.600+0000 7f3131c36700 1 -- 172.21.15.152:0/2516741971 wait complete. 2024-04-02T04:50:17.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:16 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:17.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:16 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:17.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:16 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd/host:smithi152", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:50:17.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:16 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:17.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:16 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi152", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-02T04:50:17.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:16 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi152", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-02T04:50:17.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:16 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:50:17.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:16 smithi152 ceph-mon[29073]: Deploying daemon crash.smithi152 on smithi152 2024-04-02T04:50:17.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:16 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:17.971 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi154 2024-04-02T04:50:17.971 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-04-02T04:50:17.971 DEBUG:teuthology.orchestra.run.smithi154:> dd of=/etc/ceph/ceph.conf 2024-04-02T04:50:17.990 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-04-02T04:50:17.990 DEBUG:teuthology.orchestra.run.smithi154:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:50:18.006 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:17 smithi152 ceph-mon[29073]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:50:18.047 INFO:tasks.cephadm:Adding host smithi154 to orchestrator... 2024-04-02T04:50:18.047 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph orch host add smithi154 2024-04-02T04:50:18.500 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:50:19.561 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:19 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:19.562 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:19 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:19.562 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:19 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:19.562 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:19 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:19.562 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:19 smithi152 ceph-mon[29073]: Deploying daemon node-exporter.smithi152 on smithi152 2024-04-02T04:50:20.248 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.242+0000 7f7355456700 1 -- 172.21.15.152:0/2118637618 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f73500fca60 msgr2=0x7f73500fce70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:20.248 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.242+0000 7f7355456700 1 --2- 172.21.15.152:0/2118637618 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f73500fca60 0x7f73500fce70 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 crypto rx=0x7f73440048c0 tx=0x7f7344009e30 comp rx=0 tx=0).stop 2024-04-02T04:50:20.248 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.242+0000 7f7355456700 1 -- 172.21.15.152:0/2118637618 shutdown_connections 2024-04-02T04:50:20.248 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.242+0000 7f7355456700 1 --2- 172.21.15.152:0/2118637618 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f73500fca60 0x7f73500fce70 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:20.248 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.242+0000 7f7355456700 1 -- 172.21.15.152:0/2118637618 >> 172.21.15.152:0/2118637618 conn(0x7f73500f81c0 msgr2=0x7f73500fa5f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:20.248 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.242+0000 7f7355456700 1 -- 172.21.15.152:0/2118637618 shutdown_connections 2024-04-02T04:50:20.248 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.242+0000 7f7355456700 1 -- 172.21.15.152:0/2118637618 wait complete. 2024-04-02T04:50:20.248 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.243+0000 7f7355456700 1 Processor -- start 2024-04-02T04:50:20.248 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.243+0000 7f7355456700 1 -- start start 2024-04-02T04:50:20.248 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.243+0000 7f7355456700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f73500fca60 0x7f7350071d60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:20.248 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.243+0000 7f7355456700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f73500722a0 con 0x7f73500fca60 2024-04-02T04:50:20.248 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.243+0000 7f734effd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f73500fca60 0x7f7350071d60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:20.249 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.243+0000 7f734effd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f73500fca60 0x7f7350071d60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:38256/0 (socket says 172.21.15.152:38256) 2024-04-02T04:50:20.249 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.243+0000 7f734effd700 1 -- 172.21.15.152:0/2974121451 learned_addr learned my addr 172.21.15.152:0/2974121451 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:50:20.249 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.244+0000 7f734effd700 1 -- 172.21.15.152:0/2974121451 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7344005040 con 0x7f73500fca60 2024-04-02T04:50:20.249 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.244+0000 7f734effd700 1 --2- 172.21.15.152:0/2974121451 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f73500fca60 0x7f7350071d60 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f73440096b0 tx=0x7f7344009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:20.249 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.244+0000 7f73377fe700 1 -- 172.21.15.152:0/2974121451 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f734400bed0 con 0x7f73500fca60 2024-04-02T04:50:20.249 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.244+0000 7f73377fe700 1 -- 172.21.15.152:0/2974121451 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f734400b5c0 con 0x7f73500fca60 2024-04-02T04:50:20.249 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.244+0000 7f7355456700 1 -- 172.21.15.152:0/2974121451 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7350070370 con 0x7f73500fca60 2024-04-02T04:50:20.249 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.244+0000 7f73377fe700 1 -- 172.21.15.152:0/2974121451 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f734401c5b0 con 0x7f73500fca60 2024-04-02T04:50:20.249 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.244+0000 7f7355456700 1 -- 172.21.15.152:0/2974121451 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7350070860 con 0x7f73500fca60 2024-04-02T04:50:20.250 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.245+0000 7f73377fe700 1 -- 172.21.15.152:0/2974121451 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 13) v1 ==== 43425+0+0 (secure 0 0 0) 0x7f734401c820 con 0x7f73500fca60 2024-04-02T04:50:20.250 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.245+0000 7f7355456700 1 -- 172.21.15.152:0/2974121451 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f735005f5f0 con 0x7f73500fca60 2024-04-02T04:50:20.250 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.245+0000 7f73377fe700 1 --2- 172.21.15.152:0/2974121451 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f7338036880 0x7f7338038d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:20.250 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.245+0000 7f73377fe700 1 -- 172.21.15.152:0/2974121451 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1151+0+0 (secure 0 0 0) 0x7f7344049a60 con 0x7f73500fca60 2024-04-02T04:50:20.250 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.245+0000 7f734e7fc700 1 --2- 172.21.15.152:0/2974121451 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f7338036880 0x7f7338038d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:20.250 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.246+0000 7f734e7fc700 1 --2- 172.21.15.152:0/2974121451 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f7338036880 0x7f7338038d30 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f7340009990 tx=0x7f7340006e40 comp rx=0 tx=0).ready entity=mgr.14160 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:20.254 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.251+0000 7f73377fe700 1 -- 172.21.15.152:0/2974121451 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f7344023300 con 0x7f73500fca60 2024-04-02T04:50:20.482 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:20.478+0000 7f7355456700 1 -- 172.21.15.152:0/2974121451 --> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] -- mgr_command(tid 0: {"prefix": "orch host add", "hostname": "smithi154", "target": ["mon-mgr", ""]}) v1 -- 0x7f73501064b0 con 0x7f7338036880 2024-04-02T04:50:21.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:21 smithi152 ceph-mon[29073]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi154", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:50:21.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:21 smithi152 ceph-mon[29073]: Deploying cephadm binary to smithi154 2024-04-02T04:50:22.618 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:22.614+0000 7f73377fe700 1 -- 172.21.15.152:0/2974121451 <== mgr.14160 v2:172.21.15.152:6800/2497421829 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+49 (secure 0 0 0) 0x7f73501064b0 con 0x7f7338036880 2024-04-02T04:50:22.618 INFO:teuthology.orchestra.run.smithi152.stdout:Added host 'smithi154' with addr '172.21.15.154' 2024-04-02T04:50:22.620 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:22.617+0000 7f7355456700 1 -- 172.21.15.152:0/2974121451 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f7338036880 msgr2=0x7f7338038d30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:22.620 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:22.617+0000 7f7355456700 1 --2- 172.21.15.152:0/2974121451 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f7338036880 0x7f7338038d30 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7f7340009990 tx=0x7f7340006e40 comp rx=0 tx=0).stop 2024-04-02T04:50:22.620 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:22.617+0000 7f7355456700 1 -- 172.21.15.152:0/2974121451 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f73500fca60 msgr2=0x7f7350071d60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:22.620 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:22.617+0000 7f7355456700 1 --2- 172.21.15.152:0/2974121451 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f73500fca60 0x7f7350071d60 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7f73440096b0 tx=0x7f7344009e30 comp rx=0 tx=0).stop 2024-04-02T04:50:22.620 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:22.617+0000 7f7355456700 1 -- 172.21.15.152:0/2974121451 shutdown_connections 2024-04-02T04:50:22.620 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:22.617+0000 7f7355456700 1 --2- 172.21.15.152:0/2974121451 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f7338036880 0x7f7338038d30 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:22.621 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:22.617+0000 7f7355456700 1 --2- 172.21.15.152:0/2974121451 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f73500fca60 0x7f7350071d60 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:22.621 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:22.617+0000 7f7355456700 1 -- 172.21.15.152:0/2974121451 >> 172.21.15.152:0/2974121451 conn(0x7f73500f81c0 msgr2=0x7f7350103ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:22.621 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:22.617+0000 7f7355456700 1 -- 172.21.15.152:0/2974121451 shutdown_connections 2024-04-02T04:50:22.621 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:22.617+0000 7f7355456700 1 -- 172.21.15.152:0/2974121451 wait complete. 2024-04-02T04:50:23.189 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:22 smithi152 ceph-mon[29073]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:50:23.190 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:22 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:23.190 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:22 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:23.683 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph orch host ls --format=json 2024-04-02T04:50:23.967 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:50:24.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:24 smithi152 ceph-mon[29073]: Added host smithi154 2024-04-02T04:50:24.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:24 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:24.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:24 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:24.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:24 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:24.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:24 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:24.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:24 smithi152 ceph-mon[29073]: Deploying daemon alertmanager.smithi152 on smithi152 2024-04-02T04:50:25.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:25 smithi152 ceph-mon[29073]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:50:26.011 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.819+0000 7febe81fb700 1 -- 172.21.15.152:0/175917760 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7febe00fb980 msgr2=0x7febe00fbd90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:26.011 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.819+0000 7febe81fb700 1 --2- 172.21.15.152:0/175917760 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7febe00fb980 0x7febe00fbd90 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7febdc0048c0 tx=0x7febdc009e30 comp rx=0 tx=0).stop 2024-04-02T04:50:26.011 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.819+0000 7febe81fb700 1 -- 172.21.15.152:0/175917760 shutdown_connections 2024-04-02T04:50:26.011 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.819+0000 7febe81fb700 1 --2- 172.21.15.152:0/175917760 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7febe00fb980 0x7febe00fbd90 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:26.011 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.819+0000 7febe81fb700 1 -- 172.21.15.152:0/175917760 >> 172.21.15.152:0/175917760 conn(0x7febe00f70e0 msgr2=0x7febe00f9510 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:26.011 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.819+0000 7febe81fb700 1 -- 172.21.15.152:0/175917760 shutdown_connections 2024-04-02T04:50:26.011 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.820+0000 7febe81fb700 1 -- 172.21.15.152:0/175917760 wait complete. 2024-04-02T04:50:26.011 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.820+0000 7febe81fb700 1 Processor -- start 2024-04-02T04:50:26.011 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.820+0000 7febe81fb700 1 -- start start 2024-04-02T04:50:26.011 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.820+0000 7febe81fb700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7febe00fb980 0x7febe010bf80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:26.011 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.820+0000 7febe81fb700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7febe010c4c0 con 0x7febe00fb980 2024-04-02T04:50:26.012 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.820+0000 7febe5f97700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7febe00fb980 0x7febe010bf80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:26.012 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.820+0000 7febe5f97700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7febe00fb980 0x7febe010bf80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:38042/0 (socket says 172.21.15.152:38042) 2024-04-02T04:50:26.012 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.820+0000 7febe5f97700 1 -- 172.21.15.152:0/1458709699 learned_addr learned my addr 172.21.15.152:0/1458709699 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:50:26.012 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.821+0000 7febe5f97700 1 -- 172.21.15.152:0/1458709699 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7febdc005040 con 0x7febe00fb980 2024-04-02T04:50:26.012 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.821+0000 7febe5f97700 1 --2- 172.21.15.152:0/1458709699 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7febe00fb980 0x7febe010bf80 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7febdc009680 tx=0x7febdc009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:26.012 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.821+0000 7febd67fc700 1 -- 172.21.15.152:0/1458709699 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7febdc00b5c0 con 0x7febe00fb980 2024-04-02T04:50:26.012 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.821+0000 7febd67fc700 1 -- 172.21.15.152:0/1458709699 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7febdc00be70 con 0x7febe00fb980 2024-04-02T04:50:26.012 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.821+0000 7febe81fb700 1 -- 172.21.15.152:0/1458709699 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7febe010c6c0 con 0x7febe00fb980 2024-04-02T04:50:26.012 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.821+0000 7febd67fc700 1 -- 172.21.15.152:0/1458709699 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7febdc0135a0 con 0x7febe00fb980 2024-04-02T04:50:26.012 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.821+0000 7febe81fb700 1 -- 172.21.15.152:0/1458709699 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7febe010d230 con 0x7febe00fb980 2024-04-02T04:50:26.012 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.822+0000 7febd67fc700 1 -- 172.21.15.152:0/1458709699 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 13) v1 ==== 43425+0+0 (secure 0 0 0) 0x7febdc013810 con 0x7febe00fb980 2024-04-02T04:50:26.012 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.822+0000 7febd67fc700 1 --2- 172.21.15.152:0/1458709699 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7febcc0368d0 0x7febcc038d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:26.012 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.822+0000 7febd67fc700 1 -- 172.21.15.152:0/1458709699 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1151+0+0 (secure 0 0 0) 0x7febdc049350 con 0x7febe00fb980 2024-04-02T04:50:26.012 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.822+0000 7febe5796700 1 --2- 172.21.15.152:0/1458709699 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7febcc0368d0 0x7febcc038d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:26.013 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.822+0000 7febe81fb700 1 -- 172.21.15.152:0/1458709699 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7febe0101e90 con 0x7febe00fb980 2024-04-02T04:50:26.013 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.823+0000 7febe5796700 1 --2- 172.21.15.152:0/1458709699 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7febcc0368d0 0x7febcc038d80 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7febd0009990 tx=0x7febd0006e40 comp rx=0 tx=0).ready entity=mgr.14160 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:26.013 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:25.827+0000 7febd67fc700 1 -- 172.21.15.152:0/1458709699 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7febdc021030 con 0x7febe00fb980 2024-04-02T04:50:26.050 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:26.047+0000 7febe81fb700 1 -- 172.21.15.152:0/1458709699 --> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] -- mgr_command(tid 0: {"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7febe00008d0 con 0x7febcc0368d0 2024-04-02T04:50:26.052 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:26.048+0000 7febd67fc700 1 -- 172.21.15.152:0/1458709699 <== mgr.14160 v2:172.21.15.152:6800/2497421829 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+161 (secure 0 0 0) 0x7febe00008d0 con 0x7febcc0368d0 2024-04-02T04:50:26.052 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:50:26.052 INFO:teuthology.orchestra.run.smithi152.stdout:[{"addr": "172.21.15.152", "hostname": "smithi152", "labels": [], "status": ""}, {"addr": "172.21.15.154", "hostname": "smithi154", "labels": [], "status": ""}] 2024-04-02T04:50:26.054 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:26.050+0000 7febe81fb700 1 -- 172.21.15.152:0/1458709699 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7febcc0368d0 msgr2=0x7febcc038d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:26.054 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:26.050+0000 7febe81fb700 1 --2- 172.21.15.152:0/1458709699 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7febcc0368d0 0x7febcc038d80 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7febd0009990 tx=0x7febd0006e40 comp rx=0 tx=0).stop 2024-04-02T04:50:26.054 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:26.050+0000 7febe81fb700 1 -- 172.21.15.152:0/1458709699 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7febe00fb980 msgr2=0x7febe010bf80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:26.054 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:26.050+0000 7febe81fb700 1 --2- 172.21.15.152:0/1458709699 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7febe00fb980 0x7febe010bf80 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 crypto rx=0x7febdc009680 tx=0x7febdc009e30 comp rx=0 tx=0).stop 2024-04-02T04:50:26.054 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:26.050+0000 7febe81fb700 1 -- 172.21.15.152:0/1458709699 shutdown_connections 2024-04-02T04:50:26.054 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:26.050+0000 7febe81fb700 1 --2- 172.21.15.152:0/1458709699 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7febcc0368d0 0x7febcc038d80 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:26.054 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:26.050+0000 7febe81fb700 1 --2- 172.21.15.152:0/1458709699 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7febe00fb980 0x7febe010bf80 unknown :-1 s=CLOSED pgs=95 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:26.054 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:26.050+0000 7febe81fb700 1 -- 172.21.15.152:0/1458709699 >> 172.21.15.152:0/1458709699 conn(0x7febe00f70e0 msgr2=0x7febe0102ac0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:26.055 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:26.050+0000 7febe81fb700 1 -- 172.21.15.152:0/1458709699 shutdown_connections 2024-04-02T04:50:26.055 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:26.050+0000 7febe81fb700 1 -- 172.21.15.152:0/1458709699 wait complete. 2024-04-02T04:50:27.137 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-02T04:50:27.137 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph osd crush tunables default 2024-04-02T04:50:27.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:27 smithi152 ceph-mon[29073]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:50:27.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:27 smithi152 ceph-mon[29073]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T04:50:27.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:27 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:28.142 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:50:29.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:29 smithi152 ceph-mon[29073]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:50:30.902 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.898+0000 7f8e25a0c700 1 -- 172.21.15.152:0/1168245929 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8e200fe0a0 msgr2=0x7f8e200fe4b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:30.902 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.898+0000 7f8e25a0c700 1 --2- 172.21.15.152:0/1168245929 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8e200fe0a0 0x7f8e200fe4b0 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 crypto rx=0x7f8e100048c0 tx=0x7f8e10009e30 comp rx=0 tx=0).stop 2024-04-02T04:50:30.902 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.898+0000 7f8e25a0c700 1 -- 172.21.15.152:0/1168245929 shutdown_connections 2024-04-02T04:50:30.902 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.898+0000 7f8e25a0c700 1 --2- 172.21.15.152:0/1168245929 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8e200fe0a0 0x7f8e200fe4b0 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:30.902 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.898+0000 7f8e25a0c700 1 -- 172.21.15.152:0/1168245929 >> 172.21.15.152:0/1168245929 conn(0x7f8e200f6820 msgr2=0x7f8e200f8c70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:30.902 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.898+0000 7f8e25a0c700 1 -- 172.21.15.152:0/1168245929 shutdown_connections 2024-04-02T04:50:30.902 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.898+0000 7f8e25a0c700 1 -- 172.21.15.152:0/1168245929 wait complete. 2024-04-02T04:50:30.904 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.899+0000 7f8e25a0c700 1 Processor -- start 2024-04-02T04:50:30.904 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.899+0000 7f8e25a0c700 1 -- start start 2024-04-02T04:50:30.905 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.899+0000 7f8e25a0c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8e200fe0a0 0x7f8e2010d0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:30.905 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.899+0000 7f8e25a0c700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e2010d600 con 0x7f8e200fe0a0 2024-04-02T04:50:30.905 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.899+0000 7f8e1effd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8e200fe0a0 0x7f8e2010d0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:30.905 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.899+0000 7f8e1effd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8e200fe0a0 0x7f8e2010d0c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:38374/0 (socket says 172.21.15.152:38374) 2024-04-02T04:50:30.905 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.899+0000 7f8e1effd700 1 -- 172.21.15.152:0/2209820146 learned_addr learned my addr 172.21.15.152:0/2209820146 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:50:30.905 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.900+0000 7f8e1effd700 1 -- 172.21.15.152:0/2209820146 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8e10005040 con 0x7f8e200fe0a0 2024-04-02T04:50:30.905 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.900+0000 7f8e1effd700 1 --2- 172.21.15.152:0/2209820146 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8e200fe0a0 0x7f8e2010d0c0 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f8e100096b0 tx=0x7f8e10009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:30.905 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.900+0000 7f8e077fe700 1 -- 172.21.15.152:0/2209820146 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8e1000bed0 con 0x7f8e200fe0a0 2024-04-02T04:50:30.905 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.900+0000 7f8e077fe700 1 -- 172.21.15.152:0/2209820146 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f8e1000b5c0 con 0x7f8e200fe0a0 2024-04-02T04:50:30.905 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.900+0000 7f8e25a0c700 1 -- 172.21.15.152:0/2209820146 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8e2010d800 con 0x7f8e200fe0a0 2024-04-02T04:50:30.905 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.900+0000 7f8e077fe700 1 -- 172.21.15.152:0/2209820146 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8e1001c5b0 con 0x7f8e200fe0a0 2024-04-02T04:50:30.905 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.900+0000 7f8e25a0c700 1 -- 172.21.15.152:0/2209820146 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8e2010e370 con 0x7f8e200fe0a0 2024-04-02T04:50:30.906 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.901+0000 7f8e077fe700 1 -- 172.21.15.152:0/2209820146 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 13) v1 ==== 43425+0+0 (secure 0 0 0) 0x7f8e1001c820 con 0x7f8e200fe0a0 2024-04-02T04:50:30.906 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.901+0000 7f8e25a0c700 1 -- 172.21.15.152:0/2209820146 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8e20102f70 con 0x7f8e200fe0a0 2024-04-02T04:50:30.906 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.901+0000 7f8e077fe700 1 --2- 172.21.15.152:0/2209820146 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f8e08036880 0x7f8e08038d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:30.906 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.901+0000 7f8e077fe700 1 -- 172.21.15.152:0/2209820146 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1151+0+0 (secure 0 0 0) 0x7f8e10049c70 con 0x7f8e200fe0a0 2024-04-02T04:50:30.906 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.901+0000 7f8e1e7fc700 1 --2- 172.21.15.152:0/2209820146 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f8e08036880 0x7f8e08038d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:30.906 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.902+0000 7f8e1e7fc700 1 --2- 172.21.15.152:0/2209820146 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f8e08036880 0x7f8e08038d30 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f8e14009990 tx=0x7f8e14006e40 comp rx=0 tx=0).ready entity=mgr.14160 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:30.909 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:30.906+0000 7f8e077fe700 1 -- 172.21.15.152:0/2209820146 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f8e10023300 con 0x7f8e200fe0a0 2024-04-02T04:50:31.104 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:31.101+0000 7f8e25a0c700 1 -- 172.21.15.152:0/2209820146 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "osd crush tunables", "profile": "default"} v 0) v1 -- 0x7f8e2010ed80 con 0x7f8e200fe0a0 2024-04-02T04:50:31.282 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:31.279+0000 7f8e077fe700 1 -- 172.21.15.152:0/2209820146 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "osd crush tunables", "profile": "default"}]=0 adjusted tunables profile to default v4) v1 ==== 124+0+0 (secure 0 0 0) 0x7f8e10011b10 con 0x7f8e200fe0a0 2024-04-02T04:50:31.285 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:31.281+0000 7f8e25a0c700 1 -- 172.21.15.152:0/2209820146 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f8e08036880 msgr2=0x7f8e08038d30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:31.285 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:31.281+0000 7f8e25a0c700 1 --2- 172.21.15.152:0/2209820146 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f8e08036880 0x7f8e08038d30 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 crypto rx=0x7f8e14009990 tx=0x7f8e14006e40 comp rx=0 tx=0).stop 2024-04-02T04:50:31.285 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:31.281+0000 7f8e25a0c700 1 -- 172.21.15.152:0/2209820146 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8e200fe0a0 msgr2=0x7f8e2010d0c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:31.285 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:31.281+0000 7f8e25a0c700 1 --2- 172.21.15.152:0/2209820146 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8e200fe0a0 0x7f8e2010d0c0 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f8e100096b0 tx=0x7f8e10009e30 comp rx=0 tx=0).stop 2024-04-02T04:50:31.285 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:31.282+0000 7f8e25a0c700 1 -- 172.21.15.152:0/2209820146 shutdown_connections 2024-04-02T04:50:31.286 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:31.282+0000 7f8e25a0c700 1 --2- 172.21.15.152:0/2209820146 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f8e08036880 0x7f8e08038d30 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:31.286 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:31.282+0000 7f8e25a0c700 1 --2- 172.21.15.152:0/2209820146 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8e200fe0a0 0x7f8e2010d0c0 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:31.286 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:31.282+0000 7f8e25a0c700 1 -- 172.21.15.152:0/2209820146 >> 172.21.15.152:0/2209820146 conn(0x7f8e200f6820 msgr2=0x7f8e20103ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:31.286 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:31.282+0000 7f8e25a0c700 1 -- 172.21.15.152:0/2209820146 shutdown_connections 2024-04-02T04:50:31.286 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:50:31.282+0000 7f8e25a0c700 1 -- 172.21.15.152:0/2209820146 wait complete. 2024-04-02T04:50:31.286 INFO:teuthology.orchestra.run.smithi152.stderr:adjusted tunables profile to default 2024-04-02T04:50:31.690 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:31 smithi152 ceph-mon[29073]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:50:31.690 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:31 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/2209820146' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-02T04:50:32.731 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:32 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/2209820146' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-02T04:50:32.731 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:32 smithi152 ceph-mon[29073]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T04:50:33.411 INFO:tasks.cephadm:Adding mon.smithi152 on smithi152 2024-04-02T04:50:33.411 INFO:tasks.cephadm:Adding mon.smithi154 on smithi154 2024-04-02T04:50:33.411 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph orch apply mon '2;smithi152:172.21.15.152=smithi152;smithi154:172.21.15.154=smithi154' 2024-04-02T04:50:33.577 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:33 smithi152 ceph-mon[29073]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:50:33.660 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /etc/ceph/ceph.conf 2024-04-02T04:50:33.708 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /etc/ceph/ceph.conf 2024-04-02T04:50:35.260 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.256+0000 7fd4e14ca700 1 -- 172.21.15.154:0/2995751930 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd4dc0fca80 msgr2=0x7fd4dc0fce90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:35.260 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.256+0000 7fd4e14ca700 1 --2- 172.21.15.154:0/2995751930 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd4dc0fca80 0x7fd4dc0fce90 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 crypto rx=0x7fd4d4009c60 tx=0x7fd4d4004d10 comp rx=0 tx=0).stop 2024-04-02T04:50:35.261 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.256+0000 7fd4e14ca700 1 -- 172.21.15.154:0/2995751930 shutdown_connections 2024-04-02T04:50:35.261 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.256+0000 7fd4e14ca700 1 --2- 172.21.15.154:0/2995751930 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd4dc0fca80 0x7fd4dc0fce90 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:35.261 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.256+0000 7fd4e14ca700 1 -- 172.21.15.154:0/2995751930 >> 172.21.15.154:0/2995751930 conn(0x7fd4dc0f8220 msgr2=0x7fd4dc0fa630 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:35.261 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.256+0000 7fd4e14ca700 1 -- 172.21.15.154:0/2995751930 shutdown_connections 2024-04-02T04:50:35.261 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.256+0000 7fd4e14ca700 1 -- 172.21.15.154:0/2995751930 wait complete. 2024-04-02T04:50:35.266 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.257+0000 7fd4e14ca700 1 Processor -- start 2024-04-02T04:50:35.267 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.257+0000 7fd4e14ca700 1 -- start start 2024-04-02T04:50:35.267 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.257+0000 7fd4e14ca700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd4dc0fca80 0x7fd4dc10d090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:35.267 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.257+0000 7fd4e14ca700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd4dc10d5d0 con 0x7fd4dc0fca80 2024-04-02T04:50:35.267 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.257+0000 7fd4daffd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd4dc0fca80 0x7fd4dc10d090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:35.267 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.257+0000 7fd4daffd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd4dc0fca80 0x7fd4dc10d090 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.154:54964/0 (socket says 172.21.15.154:54964) 2024-04-02T04:50:35.267 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.257+0000 7fd4daffd700 1 -- 172.21.15.154:0/3662679652 learned_addr learned my addr 172.21.15.154:0/3662679652 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-04-02T04:50:35.267 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.257+0000 7fd4daffd700 1 -- 172.21.15.154:0/3662679652 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd4d4009940 con 0x7fd4dc0fca80 2024-04-02T04:50:35.267 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.257+0000 7fd4daffd700 1 --2- 172.21.15.154:0/3662679652 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd4dc0fca80 0x7fd4dc10d090 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7fd4d4004e60 tx=0x7fd4d4004d10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:35.267 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.257+0000 7fd4d37fe700 1 -- 172.21.15.154:0/3662679652 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fd4d4012410 con 0x7fd4dc0fca80 2024-04-02T04:50:35.267 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.257+0000 7fd4d37fe700 1 -- 172.21.15.154:0/3662679652 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fd4d4012a40 con 0x7fd4dc0fca80 2024-04-02T04:50:35.267 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.257+0000 7fd4e14ca700 1 -- 172.21.15.154:0/3662679652 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd4dc10d7d0 con 0x7fd4dc0fca80 2024-04-02T04:50:35.267 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.257+0000 7fd4d37fe700 1 -- 172.21.15.154:0/3662679652 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fd4d401ba40 con 0x7fd4dc0fca80 2024-04-02T04:50:35.267 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.258+0000 7fd4e14ca700 1 -- 172.21.15.154:0/3662679652 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd4dc10e340 con 0x7fd4dc0fca80 2024-04-02T04:50:35.268 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.258+0000 7fd4d37fe700 1 -- 172.21.15.154:0/3662679652 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 13) v1 ==== 43425+0+0 (secure 0 0 0) 0x7fd4d4019040 con 0x7fd4dc0fca80 2024-04-02T04:50:35.268 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.258+0000 7fd4e14ca700 1 -- 172.21.15.154:0/3662679652 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd4dc102f90 con 0x7fd4dc0fca80 2024-04-02T04:50:35.268 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.259+0000 7fd4d37fe700 1 --2- 172.21.15.154:0/3662679652 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fd4c8036880 0x7fd4c8038d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:35.268 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.259+0000 7fd4d37fe700 1 -- 172.21.15.154:0/3662679652 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1151+0+0 (secure 0 0 0) 0x7fd4d4049cf0 con 0x7fd4dc0fca80 2024-04-02T04:50:35.268 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.259+0000 7fd4da7fc700 1 --2- 172.21.15.154:0/3662679652 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fd4c8036880 0x7fd4c8038d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:35.268 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.259+0000 7fd4da7fc700 1 --2- 172.21.15.154:0/3662679652 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fd4c8036880 0x7fd4c8038d30 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7fd4c4009990 tx=0x7fd4c4006e40 comp rx=0 tx=0).ready entity=mgr.14160 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:35.268 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.263+0000 7fd4d37fe700 1 -- 172.21.15.154:0/3662679652 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fd4d4022b90 con 0x7fd4dc0fca80 2024-04-02T04:50:35.460 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.455+0000 7fd4e14ca700 1 -- 172.21.15.154:0/3662679652 --> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi152:172.21.15.152=smithi152;smithi154:172.21.15.154=smithi154", "target": ["mon-mgr", ""]}) v1 -- 0x7fd4dc10e630 con 0x7fd4c8036880 2024-04-02T04:50:35.465 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.461+0000 7fd4d37fe700 1 -- 172.21.15.154:0/3662679652 <== mgr.14160 v2:172.21.15.152:6800/2497421829 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+24 (secure 0 0 0) 0x7fd4dc10e630 con 0x7fd4c8036880 2024-04-02T04:50:35.465 INFO:teuthology.orchestra.run.smithi154.stdout:Scheduled mon update... 2024-04-02T04:50:35.467 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.463+0000 7fd4e14ca700 1 -- 172.21.15.154:0/3662679652 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fd4c8036880 msgr2=0x7fd4c8038d30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:35.467 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.463+0000 7fd4e14ca700 1 --2- 172.21.15.154:0/3662679652 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fd4c8036880 0x7fd4c8038d30 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 crypto rx=0x7fd4c4009990 tx=0x7fd4c4006e40 comp rx=0 tx=0).stop 2024-04-02T04:50:35.468 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.463+0000 7fd4e14ca700 1 -- 172.21.15.154:0/3662679652 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd4dc0fca80 msgr2=0x7fd4dc10d090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:35.468 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.463+0000 7fd4e14ca700 1 --2- 172.21.15.154:0/3662679652 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd4dc0fca80 0x7fd4dc10d090 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 crypto rx=0x7fd4d4004e60 tx=0x7fd4d4004d10 comp rx=0 tx=0).stop 2024-04-02T04:50:35.468 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.463+0000 7fd4e14ca700 1 -- 172.21.15.154:0/3662679652 shutdown_connections 2024-04-02T04:50:35.468 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.463+0000 7fd4e14ca700 1 --2- 172.21.15.154:0/3662679652 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fd4c8036880 0x7fd4c8038d30 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:35.468 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.463+0000 7fd4e14ca700 1 --2- 172.21.15.154:0/3662679652 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd4dc0fca80 0x7fd4dc10d090 unknown :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:35.468 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.463+0000 7fd4e14ca700 1 -- 172.21.15.154:0/3662679652 >> 172.21.15.154:0/3662679652 conn(0x7fd4dc0f8220 msgr2=0x7fd4dc103bc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:35.468 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.463+0000 7fd4e14ca700 1 -- 172.21.15.154:0/3662679652 shutdown_connections 2024-04-02T04:50:35.468 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:35.463+0000 7fd4e14ca700 1 -- 172.21.15.154:0/3662679652 wait complete. 2024-04-02T04:50:35.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:35 smithi152 ceph-mon[29073]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:50:35.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:35 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:35.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:35 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:35.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:35 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:35.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:35 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:35.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:35 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:35.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:35 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:35.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:35 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T04:50:35.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:35 smithi152 ceph-mon[29073]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T04:50:35.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:35 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:35.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:35 smithi152 ceph-mon[29073]: Deploying daemon grafana.smithi152 on smithi152 2024-04-02T04:50:36.012 DEBUG:teuthology.orchestra.run.smithi154:mon.smithi154> sudo journalctl -f -n 0 -u ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4@mon.smithi154.service 2024-04-02T04:50:36.015 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T04:50:36.015 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T04:50:36.291 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /etc/ceph/ceph.conf 2024-04-02T04:50:36.338 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /etc/ceph/ceph.conf 2024-04-02T04:50:36.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:36 smithi152 ceph-mon[29073]: from='client.14192 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi152:172.21.15.152=smithi152;smithi154:172.21.15.154=smithi154", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:50:36.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:36 smithi152 ceph-mon[29073]: Saving service mon spec with placement smithi152:172.21.15.152=smithi152;smithi154:172.21.15.154=smithi154;count:2 2024-04-02T04:50:36.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:36 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:36.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:36 smithi152 ceph-mon[29073]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:50:37.780 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.775+0000 7fdd42039700 1 -- 172.21.15.154:0/1275698839 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fdd3c0fca50 msgr2=0x7fdd3c0fce60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:37.780 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.775+0000 7fdd42039700 1 --2- 172.21.15.154:0/1275698839 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fdd3c0fca50 0x7fdd3c0fce60 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7fdd2c0048c0 tx=0x7fdd2c009e30 comp rx=0 tx=0).stop 2024-04-02T04:50:37.780 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.775+0000 7fdd42039700 1 -- 172.21.15.154:0/1275698839 shutdown_connections 2024-04-02T04:50:37.780 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.775+0000 7fdd42039700 1 --2- 172.21.15.154:0/1275698839 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fdd3c0fca50 0x7fdd3c0fce60 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:37.780 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.775+0000 7fdd42039700 1 -- 172.21.15.154:0/1275698839 >> 172.21.15.154:0/1275698839 conn(0x7fdd3c0f8190 msgr2=0x7fdd3c0fa5e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:37.780 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.775+0000 7fdd42039700 1 -- 172.21.15.154:0/1275698839 shutdown_connections 2024-04-02T04:50:37.780 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.775+0000 7fdd42039700 1 -- 172.21.15.154:0/1275698839 wait complete. 2024-04-02T04:50:37.780 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.775+0000 7fdd42039700 1 Processor -- start 2024-04-02T04:50:37.780 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.776+0000 7fdd42039700 1 -- start start 2024-04-02T04:50:37.781 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.776+0000 7fdd42039700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fdd3c0fca50 0x7fdd3c10f2c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:37.781 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.776+0000 7fdd42039700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdd3c10f800 con 0x7fdd3c0fca50 2024-04-02T04:50:37.782 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.776+0000 7fdd3b7fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fdd3c0fca50 0x7fdd3c10f2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:37.782 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.776+0000 7fdd3b7fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fdd3c0fca50 0x7fdd3c10f2c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.154:54978/0 (socket says 172.21.15.154:54978) 2024-04-02T04:50:37.782 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.776+0000 7fdd3b7fe700 1 -- 172.21.15.154:0/58708382 learned_addr learned my addr 172.21.15.154:0/58708382 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-04-02T04:50:37.782 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.776+0000 7fdd3b7fe700 1 -- 172.21.15.154:0/58708382 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdd2c005040 con 0x7fdd3c0fca50 2024-04-02T04:50:37.782 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.776+0000 7fdd3b7fe700 1 --2- 172.21.15.154:0/58708382 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fdd3c0fca50 0x7fdd3c10f2c0 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7fdd2c0096b0 tx=0x7fdd2c009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:37.782 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.777+0000 7fdd23fff700 1 -- 172.21.15.154:0/58708382 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fdd2c00bed0 con 0x7fdd3c0fca50 2024-04-02T04:50:37.782 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.777+0000 7fdd42039700 1 -- 172.21.15.154:0/58708382 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdd3c10fa00 con 0x7fdd3c0fca50 2024-04-02T04:50:37.782 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.777+0000 7fdd23fff700 1 -- 172.21.15.154:0/58708382 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fdd2c00b5c0 con 0x7fdd3c0fca50 2024-04-02T04:50:37.782 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.777+0000 7fdd23fff700 1 -- 172.21.15.154:0/58708382 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fdd2c01c5b0 con 0x7fdd3c0fca50 2024-04-02T04:50:37.782 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.777+0000 7fdd42039700 1 -- 172.21.15.154:0/58708382 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdd3c110570 con 0x7fdd3c0fca50 2024-04-02T04:50:37.783 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.778+0000 7fdd23fff700 1 -- 172.21.15.154:0/58708382 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 13) v1 ==== 43425+0+0 (secure 0 0 0) 0x7fdd2c01c710 con 0x7fdd3c0fca50 2024-04-02T04:50:37.783 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.778+0000 7fdd42039700 1 -- 172.21.15.154:0/58708382 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdd3c102f60 con 0x7fdd3c0fca50 2024-04-02T04:50:37.783 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.778+0000 7fdd23fff700 1 --2- 172.21.15.154:0/58708382 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fdd240368d0 0x7fdd24038d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:37.783 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.778+0000 7fdd23fff700 1 -- 172.21.15.154:0/58708382 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1151+0+0 (secure 0 0 0) 0x7fdd2c049b70 con 0x7fdd3c0fca50 2024-04-02T04:50:37.783 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.778+0000 7fdd3affd700 1 --2- 172.21.15.154:0/58708382 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fdd240368d0 0x7fdd24038d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:37.784 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.779+0000 7fdd3affd700 1 --2- 172.21.15.154:0/58708382 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fdd240368d0 0x7fdd24038d80 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7fdd30009990 tx=0x7fdd30006e40 comp rx=0 tx=0).ready entity=mgr.14160 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:37.791 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:37.787+0000 7fdd23fff700 1 -- 172.21.15.154:0/58708382 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fdd2c025400 con 0x7fdd3c0fca50 2024-04-02T04:50:38.064 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:38.059+0000 7fdd42039700 1 -- 172.21.15.154:0/58708382 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fdd3c1107b0 con 0x7fdd3c0fca50 2024-04-02T04:50:38.064 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:38.059+0000 7fdd23fff700 1 -- 172.21.15.154:0/58708382 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+743 (secure 0 0 0) 0x7fdd2c013420 con 0x7fdd3c0fca50 2024-04-02T04:50:38.064 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:50:38.064 INFO:teuthology.orchestra.run.smithi154.stdout:{"epoch":1,"fsid":"2e0055d0-f0ac-11ee-b647-cb9ed24678a4","modified":"2024-04-02T04:48:56.938040Z","created":"2024-04-02T04:48:56.938040Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi152","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:3300","nonce":0},{"type":"v1","addr":"172.21.15.152:6789","nonce":0}]},"addr":"172.21.15.152:6789/0","public_addr":"172.21.15.152:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T04:50:38.066 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:38.062+0000 7fdd42039700 1 -- 172.21.15.154:0/58708382 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fdd240368d0 msgr2=0x7fdd24038d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:38.066 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:38.062+0000 7fdd42039700 1 --2- 172.21.15.154:0/58708382 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fdd240368d0 0x7fdd24038d80 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 crypto rx=0x7fdd30009990 tx=0x7fdd30006e40 comp rx=0 tx=0).stop 2024-04-02T04:50:38.066 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:38.062+0000 7fdd42039700 1 -- 172.21.15.154:0/58708382 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fdd3c0fca50 msgr2=0x7fdd3c10f2c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:38.066 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:38.062+0000 7fdd42039700 1 --2- 172.21.15.154:0/58708382 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fdd3c0fca50 0x7fdd3c10f2c0 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7fdd2c0096b0 tx=0x7fdd2c009e30 comp rx=0 tx=0).stop 2024-04-02T04:50:38.066 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:38.062+0000 7fdd42039700 1 -- 172.21.15.154:0/58708382 shutdown_connections 2024-04-02T04:50:38.066 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:38.062+0000 7fdd42039700 1 --2- 172.21.15.154:0/58708382 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fdd240368d0 0x7fdd24038d80 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:38.066 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:38.062+0000 7fdd42039700 1 --2- 172.21.15.154:0/58708382 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fdd3c0fca50 0x7fdd3c10f2c0 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:38.067 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:38.062+0000 7fdd42039700 1 -- 172.21.15.154:0/58708382 >> 172.21.15.154:0/58708382 conn(0x7fdd3c0f8190 msgr2=0x7fdd3c0fa5e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:38.067 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:38.062+0000 7fdd42039700 1 -- 172.21.15.154:0/58708382 shutdown_connections 2024-04-02T04:50:38.067 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:38.062+0000 7fdd42039700 1 -- 172.21.15.154:0/58708382 wait complete. 2024-04-02T04:50:38.067 INFO:teuthology.orchestra.run.smithi154.stderr:dumped monmap epoch 1 2024-04-02T04:50:38.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:37 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:39.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:38 smithi152 ceph-mon[29073]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:50:39.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:38 smithi152 ceph-mon[29073]: from='client.? 172.21.15.154:0/58708382' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:50:39.607 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T04:50:39.607 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T04:50:39.857 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /etc/ceph/ceph.conf 2024-04-02T04:50:39.904 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /etc/ceph/ceph.conf 2024-04-02T04:50:41.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:40 smithi152 ceph-mon[29073]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:50:41.407 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.402+0000 7f317f8cb700 1 -- 172.21.15.154:0/1582955540 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f31780fe0b0 msgr2=0x7f31780fe4c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:41.407 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.402+0000 7f317f8cb700 1 --2- 172.21.15.154:0/1582955540 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f31780fe0b0 0x7f31780fe4c0 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f31740048c0 tx=0x7f3174009e30 comp rx=0 tx=0).stop 2024-04-02T04:50:41.407 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.402+0000 7f317f8cb700 1 -- 172.21.15.154:0/1582955540 shutdown_connections 2024-04-02T04:50:41.408 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.402+0000 7f317f8cb700 1 --2- 172.21.15.154:0/1582955540 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f31780fe0b0 0x7f31780fe4c0 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:41.408 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.402+0000 7f317f8cb700 1 -- 172.21.15.154:0/1582955540 >> 172.21.15.154:0/1582955540 conn(0x7f31780f6850 msgr2=0x7f31780f8c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:41.408 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.402+0000 7f317f8cb700 1 -- 172.21.15.154:0/1582955540 shutdown_connections 2024-04-02T04:50:41.408 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.402+0000 7f317f8cb700 1 -- 172.21.15.154:0/1582955540 wait complete. 2024-04-02T04:50:41.408 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.403+0000 7f317f8cb700 1 Processor -- start 2024-04-02T04:50:41.408 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.403+0000 7f317f8cb700 1 -- start start 2024-04-02T04:50:41.408 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.403+0000 7f317f8cb700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f31780fe0b0 0x7f317810f300 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:41.408 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.403+0000 7f317f8cb700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f317810f840 con 0x7f31780fe0b0 2024-04-02T04:50:41.408 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.403+0000 7f317d667700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f31780fe0b0 0x7f317810f300 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:41.408 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.403+0000 7f317d667700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f31780fe0b0 0x7f317810f300 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.154:54994/0 (socket says 172.21.15.154:54994) 2024-04-02T04:50:41.408 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.403+0000 7f317d667700 1 -- 172.21.15.154:0/142823751 learned_addr learned my addr 172.21.15.154:0/142823751 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-04-02T04:50:41.408 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.403+0000 7f317d667700 1 -- 172.21.15.154:0/142823751 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3174005040 con 0x7f31780fe0b0 2024-04-02T04:50:41.410 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.403+0000 7f317d667700 1 --2- 172.21.15.154:0/142823751 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f31780fe0b0 0x7f317810f300 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f31740096b0 tx=0x7f3174009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:41.410 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.404+0000 7f316dffb700 1 -- 172.21.15.154:0/142823751 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f317400bed0 con 0x7f31780fe0b0 2024-04-02T04:50:41.410 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.404+0000 7f316dffb700 1 -- 172.21.15.154:0/142823751 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f317400b5c0 con 0x7f31780fe0b0 2024-04-02T04:50:41.410 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.404+0000 7f317f8cb700 1 -- 172.21.15.154:0/142823751 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f317810fa40 con 0x7f31780fe0b0 2024-04-02T04:50:41.410 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.404+0000 7f316dffb700 1 -- 172.21.15.154:0/142823751 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f317401c5b0 con 0x7f31780fe0b0 2024-04-02T04:50:41.410 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.404+0000 7f317f8cb700 1 -- 172.21.15.154:0/142823751 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f31781105b0 con 0x7f31780fe0b0 2024-04-02T04:50:41.410 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.405+0000 7f316dffb700 1 -- 172.21.15.154:0/142823751 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 13) v1 ==== 43425+0+0 (secure 0 0 0) 0x7f317401c820 con 0x7f31780fe0b0 2024-04-02T04:50:41.410 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.405+0000 7f317f8cb700 1 -- 172.21.15.154:0/142823751 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3178102f80 con 0x7f31780fe0b0 2024-04-02T04:50:41.410 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.405+0000 7f316dffb700 1 --2- 172.21.15.154:0/142823751 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f3164036880 0x7f3164038d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:41.411 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.405+0000 7f316dffb700 1 -- 172.21.15.154:0/142823751 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1151+0+0 (secure 0 0 0) 0x7f3174049c60 con 0x7f31780fe0b0 2024-04-02T04:50:41.411 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.405+0000 7f317ce66700 1 --2- 172.21.15.154:0/142823751 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f3164036880 0x7f3164038d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:41.411 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.405+0000 7f317ce66700 1 --2- 172.21.15.154:0/142823751 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f3164036880 0x7f3164038d30 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f3168009990 tx=0x7f3168006e40 comp rx=0 tx=0).ready entity=mgr.14160 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:41.414 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.411+0000 7f316dffb700 1 -- 172.21.15.154:0/142823751 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f3174023300 con 0x7f31780fe0b0 2024-04-02T04:50:41.677 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.673+0000 7f317f8cb700 1 -- 172.21.15.154:0/142823751 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f31781107f0 con 0x7f31780fe0b0 2024-04-02T04:50:41.678 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.673+0000 7f316dffb700 1 -- 172.21.15.154:0/142823751 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+743 (secure 0 0 0) 0x7f3174011b60 con 0x7f31780fe0b0 2024-04-02T04:50:41.678 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:50:41.678 INFO:teuthology.orchestra.run.smithi154.stdout:{"epoch":1,"fsid":"2e0055d0-f0ac-11ee-b647-cb9ed24678a4","modified":"2024-04-02T04:48:56.938040Z","created":"2024-04-02T04:48:56.938040Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi152","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:3300","nonce":0},{"type":"v1","addr":"172.21.15.152:6789","nonce":0}]},"addr":"172.21.15.152:6789/0","public_addr":"172.21.15.152:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T04:50:41.680 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.675+0000 7f317f8cb700 1 -- 172.21.15.154:0/142823751 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f3164036880 msgr2=0x7f3164038d30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:41.680 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.675+0000 7f317f8cb700 1 --2- 172.21.15.154:0/142823751 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f3164036880 0x7f3164038d30 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 crypto rx=0x7f3168009990 tx=0x7f3168006e40 comp rx=0 tx=0).stop 2024-04-02T04:50:41.680 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.675+0000 7f317f8cb700 1 -- 172.21.15.154:0/142823751 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f31780fe0b0 msgr2=0x7f317810f300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:41.680 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.675+0000 7f317f8cb700 1 --2- 172.21.15.154:0/142823751 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f31780fe0b0 0x7f317810f300 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7f31740096b0 tx=0x7f3174009e30 comp rx=0 tx=0).stop 2024-04-02T04:50:41.681 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.676+0000 7f317f8cb700 1 -- 172.21.15.154:0/142823751 shutdown_connections 2024-04-02T04:50:41.681 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.676+0000 7f317f8cb700 1 --2- 172.21.15.154:0/142823751 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f3164036880 0x7f3164038d30 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:41.681 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.676+0000 7f317f8cb700 1 --2- 172.21.15.154:0/142823751 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f31780fe0b0 0x7f317810f300 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:41.681 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.676+0000 7f317f8cb700 1 -- 172.21.15.154:0/142823751 >> 172.21.15.154:0/142823751 conn(0x7f31780f6850 msgr2=0x7f31780f8c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:41.681 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.676+0000 7f317f8cb700 1 -- 172.21.15.154:0/142823751 shutdown_connections 2024-04-02T04:50:41.681 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:41.676+0000 7f317f8cb700 1 -- 172.21.15.154:0/142823751 wait complete. 2024-04-02T04:50:41.681 INFO:teuthology.orchestra.run.smithi154.stderr:dumped monmap epoch 1 2024-04-02T04:50:42.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:41 smithi152 ceph-mon[29073]: from='client.? 172.21.15.154:0/142823751' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:50:43.229 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T04:50:43.229 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T04:50:43.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:42 smithi152 ceph-mon[29073]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:50:43.481 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /etc/ceph/ceph.conf 2024-04-02T04:50:43.527 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /etc/ceph/ceph.conf 2024-04-02T04:50:45.051 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.046+0000 7f36ec86b700 1 -- 172.21.15.154:0/2723145401 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f36e8066a40 msgr2=0x7f36e80ff650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:45.051 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.046+0000 7f36ec86b700 1 --2- 172.21.15.154:0/2723145401 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f36e8066a40 0x7f36e80ff650 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7f36d80048c0 tx=0x7f36d8009e30 comp rx=0 tx=0).stop 2024-04-02T04:50:45.051 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.046+0000 7f36ec86b700 1 -- 172.21.15.154:0/2723145401 shutdown_connections 2024-04-02T04:50:45.051 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.046+0000 7f36ec86b700 1 --2- 172.21.15.154:0/2723145401 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f36e8066a40 0x7f36e80ff650 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:45.051 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.046+0000 7f36ec86b700 1 -- 172.21.15.154:0/2723145401 >> 172.21.15.154:0/2723145401 conn(0x7f36e80f70c0 msgr2=0x7f36e80f94f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:45.051 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.046+0000 7f36ec86b700 1 -- 172.21.15.154:0/2723145401 shutdown_connections 2024-04-02T04:50:45.051 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.046+0000 7f36ec86b700 1 -- 172.21.15.154:0/2723145401 wait complete. 2024-04-02T04:50:45.051 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.046+0000 7f36ec86b700 1 Processor -- start 2024-04-02T04:50:45.051 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.047+0000 7f36ec86b700 1 -- start start 2024-04-02T04:50:45.051 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.047+0000 7f36ec86b700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f36e8066a40 0x7f36e810d0d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:45.051 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.047+0000 7f36ec86b700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f36e810d610 con 0x7f36e8066a40 2024-04-02T04:50:45.051 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.047+0000 7f36e659c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f36e8066a40 0x7f36e810d0d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:45.052 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.047+0000 7f36e659c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f36e8066a40 0x7f36e810d0d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.154:56776/0 (socket says 172.21.15.154:56776) 2024-04-02T04:50:45.052 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.047+0000 7f36e659c700 1 -- 172.21.15.154:0/3950751420 learned_addr learned my addr 172.21.15.154:0/3950751420 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-04-02T04:50:45.052 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.047+0000 7f36e659c700 1 -- 172.21.15.154:0/3950751420 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f36d8005040 con 0x7f36e8066a40 2024-04-02T04:50:45.052 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.047+0000 7f36e659c700 1 --2- 172.21.15.154:0/3950751420 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f36e8066a40 0x7f36e810d0d0 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f36d8009680 tx=0x7f36d8009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:45.052 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.048+0000 7f36d6ffd700 1 -- 172.21.15.154:0/3950751420 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f36d800bed0 con 0x7f36e8066a40 2024-04-02T04:50:45.052 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.048+0000 7f36ec86b700 1 -- 172.21.15.154:0/3950751420 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f36e810d810 con 0x7f36e8066a40 2024-04-02T04:50:45.052 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.048+0000 7f36d6ffd700 1 -- 172.21.15.154:0/3950751420 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f36d800b5c0 con 0x7f36e8066a40 2024-04-02T04:50:45.052 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.048+0000 7f36d6ffd700 1 -- 172.21.15.154:0/3950751420 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f36d801c5b0 con 0x7f36e8066a40 2024-04-02T04:50:45.052 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.048+0000 7f36ec86b700 1 -- 172.21.15.154:0/3950751420 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f36e810e380 con 0x7f36e8066a40 2024-04-02T04:50:45.053 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.048+0000 7f36ec86b700 1 -- 172.21.15.154:0/3950751420 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f36e8102f60 con 0x7f36e8066a40 2024-04-02T04:50:45.053 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.048+0000 7f36d6ffd700 1 -- 172.21.15.154:0/3950751420 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 13) v1 ==== 43425+0+0 (secure 0 0 0) 0x7f36d8013420 con 0x7f36e8066a40 2024-04-02T04:50:45.053 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.049+0000 7f36d6ffd700 1 --2- 172.21.15.154:0/3950751420 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f36d0036880 0x7f36d0038d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:45.053 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.049+0000 7f36d6ffd700 1 -- 172.21.15.154:0/3950751420 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1151+0+0 (secure 0 0 0) 0x7f36d8049a40 con 0x7f36e8066a40 2024-04-02T04:50:45.053 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.049+0000 7f36e5d9b700 1 --2- 172.21.15.154:0/3950751420 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f36d0036880 0x7f36d0038d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:45.053 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.049+0000 7f36e5d9b700 1 --2- 172.21.15.154:0/3950751420 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f36d0036880 0x7f36d0038d30 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f36dc009990 tx=0x7f36dc006e40 comp rx=0 tx=0).ready entity=mgr.14160 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:45.057 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.054+0000 7f36d6ffd700 1 -- 172.21.15.154:0/3950751420 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f36d8013a80 con 0x7f36e8066a40 2024-04-02T04:50:45.238 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:44 smithi152 ceph-mon[29073]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:50:45.319 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.316+0000 7f36ec86b700 1 -- 172.21.15.154:0/3950751420 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f36e810ed90 con 0x7f36e8066a40 2024-04-02T04:50:45.319 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.316+0000 7f36d6ffd700 1 -- 172.21.15.154:0/3950751420 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+743 (secure 0 0 0) 0x7f36d8011370 con 0x7f36e8066a40 2024-04-02T04:50:45.320 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:50:45.320 INFO:teuthology.orchestra.run.smithi154.stdout:{"epoch":1,"fsid":"2e0055d0-f0ac-11ee-b647-cb9ed24678a4","modified":"2024-04-02T04:48:56.938040Z","created":"2024-04-02T04:48:56.938040Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi152","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:3300","nonce":0},{"type":"v1","addr":"172.21.15.152:6789","nonce":0}]},"addr":"172.21.15.152:6789/0","public_addr":"172.21.15.152:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T04:50:45.322 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.318+0000 7f36ec86b700 1 -- 172.21.15.154:0/3950751420 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f36d0036880 msgr2=0x7f36d0038d30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:45.322 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.318+0000 7f36ec86b700 1 --2- 172.21.15.154:0/3950751420 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f36d0036880 0x7f36d0038d30 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7f36dc009990 tx=0x7f36dc006e40 comp rx=0 tx=0).stop 2024-04-02T04:50:45.322 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.318+0000 7f36ec86b700 1 -- 172.21.15.154:0/3950751420 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f36e8066a40 msgr2=0x7f36e810d0d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:45.322 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.318+0000 7f36ec86b700 1 --2- 172.21.15.154:0/3950751420 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f36e8066a40 0x7f36e810d0d0 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f36d8009680 tx=0x7f36d8009e30 comp rx=0 tx=0).stop 2024-04-02T04:50:45.322 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.318+0000 7f36ec86b700 1 -- 172.21.15.154:0/3950751420 shutdown_connections 2024-04-02T04:50:45.322 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.318+0000 7f36ec86b700 1 --2- 172.21.15.154:0/3950751420 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f36d0036880 0x7f36d0038d30 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:45.322 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.318+0000 7f36ec86b700 1 --2- 172.21.15.154:0/3950751420 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f36e8066a40 0x7f36e810d0d0 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:45.322 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.318+0000 7f36ec86b700 1 -- 172.21.15.154:0/3950751420 >> 172.21.15.154:0/3950751420 conn(0x7f36e80f70c0 msgr2=0x7f36e8103b90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:45.322 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.319+0000 7f36ec86b700 1 -- 172.21.15.154:0/3950751420 shutdown_connections 2024-04-02T04:50:45.323 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:45.319+0000 7f36ec86b700 1 -- 172.21.15.154:0/3950751420 wait complete. 2024-04-02T04:50:45.323 INFO:teuthology.orchestra.run.smithi154.stderr:dumped monmap epoch 1 2024-04-02T04:50:46.223 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:45 smithi152 ceph-mon[29073]: from='client.? 172.21.15.154:0/3950751420' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:50:46.908 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T04:50:46.909 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T04:50:47.161 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /etc/ceph/ceph.conf 2024-04-02T04:50:47.203 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:46 smithi152 ceph-mon[29073]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:50:47.208 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /etc/ceph/ceph.conf 2024-04-02T04:50:48.265 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:48 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:48.266 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:48 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:48.266 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:48 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:48.266 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:48 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:48.266 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:48 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:48.266 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:48 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:48.266 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:48 smithi152 ceph-mon[29073]: Deploying daemon prometheus.smithi152 on smithi152 2024-04-02T04:50:48.708 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.703+0000 7fcfbb5e1700 1 -- 172.21.15.154:0/1033856599 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcfb40fd050 msgr2=0x7fcfb40fd460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:48.709 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.703+0000 7fcfbb5e1700 1 --2- 172.21.15.154:0/1033856599 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcfb40fd050 0x7fcfb40fd460 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7fcfa4009c60 tx=0x7fcfa4004d10 comp rx=0 tx=0).stop 2024-04-02T04:50:48.709 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.703+0000 7fcfbb5e1700 1 -- 172.21.15.154:0/1033856599 shutdown_connections 2024-04-02T04:50:48.709 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.703+0000 7fcfbb5e1700 1 --2- 172.21.15.154:0/1033856599 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcfb40fd050 0x7fcfb40fd460 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:48.709 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.703+0000 7fcfbb5e1700 1 -- 172.21.15.154:0/1033856599 >> 172.21.15.154:0/1033856599 conn(0x7fcfb40f70d0 msgr2=0x7fcfb40f9500 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:48.709 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.703+0000 7fcfbb5e1700 1 -- 172.21.15.154:0/1033856599 shutdown_connections 2024-04-02T04:50:48.709 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.703+0000 7fcfbb5e1700 1 -- 172.21.15.154:0/1033856599 wait complete. 2024-04-02T04:50:48.709 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.704+0000 7fcfbb5e1700 1 Processor -- start 2024-04-02T04:50:48.709 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.704+0000 7fcfbb5e1700 1 -- start start 2024-04-02T04:50:48.709 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.704+0000 7fcfbb5e1700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcfb40fd050 0x7fcfb410bfc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:48.709 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.704+0000 7fcfbb5e1700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcfb410c500 con 0x7fcfb40fd050 2024-04-02T04:50:48.709 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.704+0000 7fcfb937d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcfb40fd050 0x7fcfb410bfc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:48.709 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.704+0000 7fcfb937d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcfb40fd050 0x7fcfb410bfc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.154:56788/0 (socket says 172.21.15.154:56788) 2024-04-02T04:50:48.709 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.704+0000 7fcfb937d700 1 -- 172.21.15.154:0/2747056202 learned_addr learned my addr 172.21.15.154:0/2747056202 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-04-02T04:50:48.709 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.704+0000 7fcfb937d700 1 -- 172.21.15.154:0/2747056202 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcfa4009940 con 0x7fcfb40fd050 2024-04-02T04:50:48.709 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.705+0000 7fcfb937d700 1 --2- 172.21.15.154:0/2747056202 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcfb40fd050 0x7fcfb410bfc0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7fcfa4009500 tx=0x7fcfa4004d10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:48.710 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.705+0000 7fcfb1ffb700 1 -- 172.21.15.154:0/2747056202 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fcfa4007c80 con 0x7fcfb40fd050 2024-04-02T04:50:48.710 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.705+0000 7fcfb1ffb700 1 -- 172.21.15.154:0/2747056202 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fcfa401d910 con 0x7fcfb40fd050 2024-04-02T04:50:48.710 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.705+0000 7fcfbb5e1700 1 -- 172.21.15.154:0/2747056202 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcfb410c700 con 0x7fcfb40fd050 2024-04-02T04:50:48.710 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.705+0000 7fcfb1ffb700 1 -- 172.21.15.154:0/2747056202 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fcfa40129d0 con 0x7fcfb40fd050 2024-04-02T04:50:48.710 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.705+0000 7fcfbb5e1700 1 -- 172.21.15.154:0/2747056202 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcfb410d270 con 0x7fcfb40fd050 2024-04-02T04:50:48.711 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.706+0000 7fcfb1ffb700 1 -- 172.21.15.154:0/2747056202 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 13) v1 ==== 43425+0+0 (secure 0 0 0) 0x7fcfa4019040 con 0x7fcfb40fd050 2024-04-02T04:50:48.711 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.706+0000 7fcfbb5e1700 1 -- 172.21.15.154:0/2747056202 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcfb4101e70 con 0x7fcfb40fd050 2024-04-02T04:50:48.711 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.706+0000 7fcfb1ffb700 1 --2- 172.21.15.154:0/2747056202 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fcf9c036880 0x7fcf9c038d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:48.711 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.706+0000 7fcfb1ffb700 1 -- 172.21.15.154:0/2747056202 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1151+0+0 (secure 0 0 0) 0x7fcfa404a400 con 0x7fcfb40fd050 2024-04-02T04:50:48.711 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.706+0000 7fcfb8b7c700 1 --2- 172.21.15.154:0/2747056202 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fcf9c036880 0x7fcf9c038d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:48.711 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.707+0000 7fcfb8b7c700 1 --2- 172.21.15.154:0/2747056202 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fcf9c036880 0x7fcf9c038d30 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7fcfa8009990 tx=0x7fcfa8006e40 comp rx=0 tx=0).ready entity=mgr.14160 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:48.717 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.714+0000 7fcfb1ffb700 1 -- 172.21.15.154:0/2747056202 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fcfa402eb40 con 0x7fcfb40fd050 2024-04-02T04:50:48.980 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.977+0000 7fcfbb5e1700 1 -- 172.21.15.154:0/2747056202 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fcfb410dbd0 con 0x7fcfb40fd050 2024-04-02T04:50:48.981 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.978+0000 7fcfb1ffb700 1 -- 172.21.15.154:0/2747056202 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+743 (secure 0 0 0) 0x7fcfa4017020 con 0x7fcfb40fd050 2024-04-02T04:50:48.981 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:50:48.981 INFO:teuthology.orchestra.run.smithi154.stdout:{"epoch":1,"fsid":"2e0055d0-f0ac-11ee-b647-cb9ed24678a4","modified":"2024-04-02T04:48:56.938040Z","created":"2024-04-02T04:48:56.938040Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi152","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:3300","nonce":0},{"type":"v1","addr":"172.21.15.152:6789","nonce":0}]},"addr":"172.21.15.152:6789/0","public_addr":"172.21.15.152:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T04:50:48.983 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.980+0000 7fcfbb5e1700 1 -- 172.21.15.154:0/2747056202 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fcf9c036880 msgr2=0x7fcf9c038d30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:48.983 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.980+0000 7fcfbb5e1700 1 --2- 172.21.15.154:0/2747056202 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fcf9c036880 0x7fcf9c038d30 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 crypto rx=0x7fcfa8009990 tx=0x7fcfa8006e40 comp rx=0 tx=0).stop 2024-04-02T04:50:48.984 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.980+0000 7fcfbb5e1700 1 -- 172.21.15.154:0/2747056202 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcfb40fd050 msgr2=0x7fcfb410bfc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:48.984 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.980+0000 7fcfbb5e1700 1 --2- 172.21.15.154:0/2747056202 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcfb40fd050 0x7fcfb410bfc0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7fcfa4009500 tx=0x7fcfa4004d10 comp rx=0 tx=0).stop 2024-04-02T04:50:48.984 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.980+0000 7fcfbb5e1700 1 -- 172.21.15.154:0/2747056202 shutdown_connections 2024-04-02T04:50:48.984 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.980+0000 7fcfbb5e1700 1 --2- 172.21.15.154:0/2747056202 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fcf9c036880 0x7fcf9c038d30 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:48.984 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.980+0000 7fcfbb5e1700 1 --2- 172.21.15.154:0/2747056202 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcfb40fd050 0x7fcfb410bfc0 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:48.984 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.980+0000 7fcfbb5e1700 1 -- 172.21.15.154:0/2747056202 >> 172.21.15.154:0/2747056202 conn(0x7fcfb40f70d0 msgr2=0x7fcfb4102aa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:48.984 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.980+0000 7fcfbb5e1700 1 -- 172.21.15.154:0/2747056202 shutdown_connections 2024-04-02T04:50:48.984 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:48.980+0000 7fcfbb5e1700 1 -- 172.21.15.154:0/2747056202 wait complete. 2024-04-02T04:50:48.984 INFO:teuthology.orchestra.run.smithi154.stderr:dumped monmap epoch 1 2024-04-02T04:50:49.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:49 smithi152 ceph-mon[29073]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:50:49.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:49 smithi152 ceph-mon[29073]: from='client.? 172.21.15.154:0/2747056202' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:50:50.529 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T04:50:50.530 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T04:50:50.781 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /etc/ceph/ceph.conf 2024-04-02T04:50:50.828 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /etc/ceph/ceph.conf 2024-04-02T04:50:51.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:51 smithi152 ceph-mon[29073]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:50:52.227 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.223+0000 7f3f8d190700 1 -- 172.21.15.154:0/1741065634 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3f880faea0 msgr2=0x7f3f880fb2b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:52.227 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.223+0000 7f3f8d190700 1 --2- 172.21.15.154:0/1741065634 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3f880faea0 0x7f3f880fb2b0 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7f3f7c0048c0 tx=0x7f3f7c009e30 comp rx=0 tx=0).stop 2024-04-02T04:50:52.228 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.223+0000 7f3f8d190700 1 -- 172.21.15.154:0/1741065634 shutdown_connections 2024-04-02T04:50:52.228 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.223+0000 7f3f8d190700 1 --2- 172.21.15.154:0/1741065634 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3f880faea0 0x7f3f880fb2b0 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:52.228 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.223+0000 7f3f8d190700 1 -- 172.21.15.154:0/1741065634 >> 172.21.15.154:0/1741065634 conn(0x7f3f880f6820 msgr2=0x7f3f880f8c70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:52.228 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.223+0000 7f3f8d190700 1 -- 172.21.15.154:0/1741065634 shutdown_connections 2024-04-02T04:50:52.228 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.223+0000 7f3f8d190700 1 -- 172.21.15.154:0/1741065634 wait complete. 2024-04-02T04:50:52.228 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.224+0000 7f3f8d190700 1 Processor -- start 2024-04-02T04:50:52.228 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.224+0000 7f3f8d190700 1 -- start start 2024-04-02T04:50:52.228 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.224+0000 7f3f8d190700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3f880faea0 0x7f3f880fd8d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:52.228 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.224+0000 7f3f8d190700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3f880fde10 con 0x7f3f880faea0 2024-04-02T04:50:52.230 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.225+0000 7f3f86d9d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3f880faea0 0x7f3f880fd8d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:52.230 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.225+0000 7f3f86d9d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3f880faea0 0x7f3f880fd8d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.154:56794/0 (socket says 172.21.15.154:56794) 2024-04-02T04:50:52.230 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.225+0000 7f3f86d9d700 1 -- 172.21.15.154:0/2162923981 learned_addr learned my addr 172.21.15.154:0/2162923981 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-04-02T04:50:52.230 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.225+0000 7f3f86d9d700 1 -- 172.21.15.154:0/2162923981 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3f7c005040 con 0x7f3f880faea0 2024-04-02T04:50:52.230 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.225+0000 7f3f86d9d700 1 --2- 172.21.15.154:0/2162923981 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3f880faea0 0x7f3f880fd8d0 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f3f7c0096b0 tx=0x7f3f7c009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:52.230 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.225+0000 7f3f6f7fe700 1 -- 172.21.15.154:0/2162923981 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f3f7c00bed0 con 0x7f3f880faea0 2024-04-02T04:50:52.230 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.225+0000 7f3f6f7fe700 1 -- 172.21.15.154:0/2162923981 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f3f7c00b5c0 con 0x7f3f880faea0 2024-04-02T04:50:52.230 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.225+0000 7f3f6f7fe700 1 -- 172.21.15.154:0/2162923981 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f3f7c01c5b0 con 0x7f3f880faea0 2024-04-02T04:50:52.231 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.225+0000 7f3f8d190700 1 -- 172.21.15.154:0/2162923981 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3f880fbee0 con 0x7f3f880faea0 2024-04-02T04:50:52.231 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.225+0000 7f3f8d190700 1 -- 172.21.15.154:0/2162923981 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3f880fc3d0 con 0x7f3f880faea0 2024-04-02T04:50:52.231 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.226+0000 7f3f6f7fe700 1 -- 172.21.15.154:0/2162923981 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 13) v1 ==== 43425+0+0 (secure 0 0 0) 0x7f3f7c01c820 con 0x7f3f880faea0 2024-04-02T04:50:52.231 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.226+0000 7f3f8d190700 1 -- 172.21.15.154:0/2162923981 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3f8805f5f0 con 0x7f3f880faea0 2024-04-02T04:50:52.231 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.226+0000 7f3f6f7fe700 1 --2- 172.21.15.154:0/2162923981 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f3f70036880 0x7f3f70038d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:52.231 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.226+0000 7f3f6f7fe700 1 -- 172.21.15.154:0/2162923981 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1151+0+0 (secure 0 0 0) 0x7f3f7c049bc0 con 0x7f3f880faea0 2024-04-02T04:50:52.231 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.226+0000 7f3f8659c700 1 --2- 172.21.15.154:0/2162923981 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f3f70036880 0x7f3f70038d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:52.231 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.227+0000 7f3f8659c700 1 --2- 172.21.15.154:0/2162923981 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f3f70036880 0x7f3f70038d30 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f3f78009990 tx=0x7f3f78006e40 comp rx=0 tx=0).ready entity=mgr.14160 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:52.234 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.231+0000 7f3f6f7fe700 1 -- 172.21.15.154:0/2162923981 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f3f7c013420 con 0x7f3f880faea0 2024-04-02T04:50:52.491 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.488+0000 7f3f8d190700 1 -- 172.21.15.154:0/2162923981 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f3f88106450 con 0x7f3f880faea0 2024-04-02T04:50:52.492 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.489+0000 7f3f6f7fe700 1 -- 172.21.15.154:0/2162923981 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+743 (secure 0 0 0) 0x7f3f7c004d30 con 0x7f3f880faea0 2024-04-02T04:50:52.492 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:50:52.492 INFO:teuthology.orchestra.run.smithi154.stdout:{"epoch":1,"fsid":"2e0055d0-f0ac-11ee-b647-cb9ed24678a4","modified":"2024-04-02T04:48:56.938040Z","created":"2024-04-02T04:48:56.938040Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi152","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:3300","nonce":0},{"type":"v1","addr":"172.21.15.152:6789","nonce":0}]},"addr":"172.21.15.152:6789/0","public_addr":"172.21.15.152:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T04:50:52.494 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.491+0000 7f3f8d190700 1 -- 172.21.15.154:0/2162923981 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f3f70036880 msgr2=0x7f3f70038d30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:52.494 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.491+0000 7f3f8d190700 1 --2- 172.21.15.154:0/2162923981 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f3f70036880 0x7f3f70038d30 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 crypto rx=0x7f3f78009990 tx=0x7f3f78006e40 comp rx=0 tx=0).stop 2024-04-02T04:50:52.495 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.491+0000 7f3f8d190700 1 -- 172.21.15.154:0/2162923981 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3f880faea0 msgr2=0x7f3f880fd8d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:52.495 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.491+0000 7f3f8d190700 1 --2- 172.21.15.154:0/2162923981 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3f880faea0 0x7f3f880fd8d0 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f3f7c0096b0 tx=0x7f3f7c009e30 comp rx=0 tx=0).stop 2024-04-02T04:50:52.495 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.491+0000 7f3f8d190700 1 -- 172.21.15.154:0/2162923981 shutdown_connections 2024-04-02T04:50:52.495 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.491+0000 7f3f8d190700 1 --2- 172.21.15.154:0/2162923981 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f3f70036880 0x7f3f70038d30 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:52.495 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.491+0000 7f3f8d190700 1 --2- 172.21.15.154:0/2162923981 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3f880faea0 0x7f3f880fd8d0 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:52.495 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.491+0000 7f3f8d190700 1 -- 172.21.15.154:0/2162923981 >> 172.21.15.154:0/2162923981 conn(0x7f3f880f6820 msgr2=0x7f3f88103bb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:52.495 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.491+0000 7f3f8d190700 1 -- 172.21.15.154:0/2162923981 shutdown_connections 2024-04-02T04:50:52.495 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:52.491+0000 7f3f8d190700 1 -- 172.21.15.154:0/2162923981 wait complete. 2024-04-02T04:50:52.495 INFO:teuthology.orchestra.run.smithi154.stderr:dumped monmap epoch 1 2024-04-02T04:50:53.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:52 smithi152 ceph-mon[29073]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:50:53.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:52 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:50:53.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:52 smithi152 ceph-mon[29073]: from='client.? 172.21.15.154:0/2162923981' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:50:54.059 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T04:50:54.059 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T04:50:54.307 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /etc/ceph/ceph.conf 2024-04-02T04:50:54.353 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /etc/ceph/ceph.conf 2024-04-02T04:50:55.086 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:54 smithi152 ceph-mon[29073]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:50:55.852 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.845+0000 7faa58f17700 1 -- 172.21.15.154:0/4049258637 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7faa540fca60 msgr2=0x7faa540fce70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:55.852 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.845+0000 7faa58f17700 1 --2- 172.21.15.154:0/4049258637 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7faa540fca60 0x7faa540fce70 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7faa44009c60 tx=0x7faa44004d10 comp rx=0 tx=0).stop 2024-04-02T04:50:55.852 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.845+0000 7faa58f17700 1 -- 172.21.15.154:0/4049258637 shutdown_connections 2024-04-02T04:50:55.852 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.845+0000 7faa58f17700 1 --2- 172.21.15.154:0/4049258637 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7faa540fca60 0x7faa540fce70 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:55.852 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.845+0000 7faa58f17700 1 -- 172.21.15.154:0/4049258637 >> 172.21.15.154:0/4049258637 conn(0x7faa540f81c0 msgr2=0x7faa540fa5f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:55.852 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.845+0000 7faa58f17700 1 -- 172.21.15.154:0/4049258637 shutdown_connections 2024-04-02T04:50:55.852 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.845+0000 7faa58f17700 1 -- 172.21.15.154:0/4049258637 wait complete. 2024-04-02T04:50:55.853 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.846+0000 7faa58f17700 1 Processor -- start 2024-04-02T04:50:55.853 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.846+0000 7faa58f17700 1 -- start start 2024-04-02T04:50:55.853 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.846+0000 7faa58f17700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7faa540fca60 0x7faa5410d0b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:55.853 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.846+0000 7faa58f17700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faa5410d5f0 con 0x7faa540fca60 2024-04-02T04:50:55.853 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.846+0000 7faa5259c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7faa540fca60 0x7faa5410d0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:55.853 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.846+0000 7faa5259c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7faa540fca60 0x7faa5410d0b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.154:55838/0 (socket says 172.21.15.154:55838) 2024-04-02T04:50:55.853 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.846+0000 7faa5259c700 1 -- 172.21.15.154:0/2183053541 learned_addr learned my addr 172.21.15.154:0/2183053541 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-04-02T04:50:55.853 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.846+0000 7faa5259c700 1 -- 172.21.15.154:0/2183053541 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faa44009940 con 0x7faa540fca60 2024-04-02T04:50:55.853 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.847+0000 7faa5259c700 1 --2- 172.21.15.154:0/2183053541 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7faa540fca60 0x7faa5410d0b0 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7faa44009500 tx=0x7faa44004d10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:55.853 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.847+0000 7faa42ffd700 1 -- 172.21.15.154:0/2183053541 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7faa44007c80 con 0x7faa540fca60 2024-04-02T04:50:55.853 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.847+0000 7faa58f17700 1 -- 172.21.15.154:0/2183053541 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7faa5410d7f0 con 0x7faa540fca60 2024-04-02T04:50:55.853 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.847+0000 7faa42ffd700 1 -- 172.21.15.154:0/2183053541 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7faa4401d910 con 0x7faa540fca60 2024-04-02T04:50:55.853 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.847+0000 7faa42ffd700 1 -- 172.21.15.154:0/2183053541 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7faa440129d0 con 0x7faa540fca60 2024-04-02T04:50:55.854 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.847+0000 7faa58f17700 1 -- 172.21.15.154:0/2183053541 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7faa5410e360 con 0x7faa540fca60 2024-04-02T04:50:55.854 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.848+0000 7faa58f17700 1 -- 172.21.15.154:0/2183053541 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7faa54102f70 con 0x7faa540fca60 2024-04-02T04:50:55.854 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.848+0000 7faa42ffd700 1 -- 172.21.15.154:0/2183053541 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 13) v1 ==== 43425+0+0 (secure 0 0 0) 0x7faa44019040 con 0x7faa540fca60 2024-04-02T04:50:55.854 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.848+0000 7faa42ffd700 1 --2- 172.21.15.154:0/2183053541 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7faa3c036880 0x7faa3c038d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:55.854 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.848+0000 7faa42ffd700 1 -- 172.21.15.154:0/2183053541 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1151+0+0 (secure 0 0 0) 0x7faa4404a450 con 0x7faa540fca60 2024-04-02T04:50:55.854 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.848+0000 7faa43fff700 1 --2- 172.21.15.154:0/2183053541 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7faa3c036880 0x7faa3c038d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:55.854 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.849+0000 7faa43fff700 1 --2- 172.21.15.154:0/2183053541 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7faa3c036880 0x7faa3c038d30 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7faa48009990 tx=0x7faa48006e40 comp rx=0 tx=0).ready entity=mgr.14160 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:55.856 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:55.853+0000 7faa42ffd700 1 -- 172.21.15.154:0/2183053541 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7faa4402eb40 con 0x7faa540fca60 2024-04-02T04:50:56.119 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:56.115+0000 7faa58f17700 1 -- 172.21.15.154:0/2183053541 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7faa5410ecc0 con 0x7faa540fca60 2024-04-02T04:50:56.119 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:56.116+0000 7faa42ffd700 1 -- 172.21.15.154:0/2183053541 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+743 (secure 0 0 0) 0x7faa44017020 con 0x7faa540fca60 2024-04-02T04:50:56.119 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:50:56.120 INFO:teuthology.orchestra.run.smithi154.stdout:{"epoch":1,"fsid":"2e0055d0-f0ac-11ee-b647-cb9ed24678a4","modified":"2024-04-02T04:48:56.938040Z","created":"2024-04-02T04:48:56.938040Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi152","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:3300","nonce":0},{"type":"v1","addr":"172.21.15.152:6789","nonce":0}]},"addr":"172.21.15.152:6789/0","public_addr":"172.21.15.152:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T04:50:56.121 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:56.118+0000 7faa58f17700 1 -- 172.21.15.154:0/2183053541 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7faa3c036880 msgr2=0x7faa3c038d30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:56.121 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:56.118+0000 7faa58f17700 1 --2- 172.21.15.154:0/2183053541 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7faa3c036880 0x7faa3c038d30 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7faa48009990 tx=0x7faa48006e40 comp rx=0 tx=0).stop 2024-04-02T04:50:56.121 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:56.118+0000 7faa58f17700 1 -- 172.21.15.154:0/2183053541 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7faa540fca60 msgr2=0x7faa5410d0b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:56.122 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:56.118+0000 7faa58f17700 1 --2- 172.21.15.154:0/2183053541 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7faa540fca60 0x7faa5410d0b0 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7faa44009500 tx=0x7faa44004d10 comp rx=0 tx=0).stop 2024-04-02T04:50:56.122 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:56.118+0000 7faa58f17700 1 -- 172.21.15.154:0/2183053541 shutdown_connections 2024-04-02T04:50:56.122 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:56.118+0000 7faa58f17700 1 --2- 172.21.15.154:0/2183053541 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7faa3c036880 0x7faa3c038d30 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:56.122 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:56.118+0000 7faa58f17700 1 --2- 172.21.15.154:0/2183053541 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7faa540fca60 0x7faa5410d0b0 unknown :-1 s=CLOSED pgs=111 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:56.122 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:56.118+0000 7faa58f17700 1 -- 172.21.15.154:0/2183053541 >> 172.21.15.154:0/2183053541 conn(0x7faa540f81c0 msgr2=0x7faa54103ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:56.122 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:56.118+0000 7faa58f17700 1 -- 172.21.15.154:0/2183053541 shutdown_connections 2024-04-02T04:50:56.122 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:56.118+0000 7faa58f17700 1 -- 172.21.15.154:0/2183053541 wait complete. 2024-04-02T04:50:56.122 INFO:teuthology.orchestra.run.smithi154.stderr:dumped monmap epoch 1 2024-04-02T04:50:57.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:56 smithi152 ceph-mon[29073]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:50:57.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:56 smithi152 ceph-mon[29073]: from='client.? 172.21.15.154:0/2183053541' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:50:57.704 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T04:50:57.705 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T04:50:57.953 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /etc/ceph/ceph.conf 2024-04-02T04:50:58.001 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /etc/ceph/ceph.conf 2024-04-02T04:50:59.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:58 smithi152 ceph-mon[29073]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:50:59.506 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.502+0000 7ff67b1ac700 1 -- 172.21.15.154:0/739655526 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff6740fb0e0 msgr2=0x7ff6740fb4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:59.507 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.502+0000 7ff67b1ac700 1 --2- 172.21.15.154:0/739655526 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff6740fb0e0 0x7ff6740fb4f0 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7ff6680048c0 tx=0x7ff668009e30 comp rx=0 tx=0).stop 2024-04-02T04:50:59.507 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.502+0000 7ff67b1ac700 1 -- 172.21.15.154:0/739655526 shutdown_connections 2024-04-02T04:50:59.507 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.502+0000 7ff67b1ac700 1 --2- 172.21.15.154:0/739655526 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff6740fb0e0 0x7ff6740fb4f0 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:59.507 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.502+0000 7ff67b1ac700 1 -- 172.21.15.154:0/739655526 >> 172.21.15.154:0/739655526 conn(0x7ff6740f6840 msgr2=0x7ff6740f8c70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:59.507 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.502+0000 7ff67b1ac700 1 -- 172.21.15.154:0/739655526 shutdown_connections 2024-04-02T04:50:59.507 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.502+0000 7ff67b1ac700 1 -- 172.21.15.154:0/739655526 wait complete. 2024-04-02T04:50:59.507 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.502+0000 7ff67b1ac700 1 Processor -- start 2024-04-02T04:50:59.507 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.502+0000 7ff67b1ac700 1 -- start start 2024-04-02T04:50:59.507 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.502+0000 7ff67b1ac700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff6740fb0e0 0x7ff6740f77c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:59.507 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.502+0000 7ff67b1ac700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff67410fde0 con 0x7ff6740fb0e0 2024-04-02T04:50:59.508 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.503+0000 7ff678f48700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff6740fb0e0 0x7ff6740f77c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:59.508 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.503+0000 7ff678f48700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff6740fb0e0 0x7ff6740f77c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.154:55848/0 (socket says 172.21.15.154:55848) 2024-04-02T04:50:59.508 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.503+0000 7ff678f48700 1 -- 172.21.15.154:0/2750843789 learned_addr learned my addr 172.21.15.154:0/2750843789 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-04-02T04:50:59.508 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.503+0000 7ff678f48700 1 -- 172.21.15.154:0/2750843789 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff668005040 con 0x7ff6740fb0e0 2024-04-02T04:50:59.508 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.503+0000 7ff678f48700 1 --2- 172.21.15.154:0/2750843789 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff6740fb0e0 0x7ff6740f77c0 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7ff668009680 tx=0x7ff668009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:59.508 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.503+0000 7ff6717fa700 1 -- 172.21.15.154:0/2750843789 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ff66800b5c0 con 0x7ff6740fb0e0 2024-04-02T04:50:59.508 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.503+0000 7ff6717fa700 1 -- 172.21.15.154:0/2750843789 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7ff66800be70 con 0x7ff6740fb0e0 2024-04-02T04:50:59.508 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.503+0000 7ff67b1ac700 1 -- 172.21.15.154:0/2750843789 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff67410ff20 con 0x7ff6740fb0e0 2024-04-02T04:50:59.509 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.503+0000 7ff6717fa700 1 -- 172.21.15.154:0/2750843789 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ff6680135a0 con 0x7ff6740fb0e0 2024-04-02T04:50:59.509 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.503+0000 7ff67b1ac700 1 -- 172.21.15.154:0/2750843789 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff674110410 con 0x7ff6740fb0e0 2024-04-02T04:50:59.509 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.504+0000 7ff6717fa700 1 -- 172.21.15.154:0/2750843789 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 13) v1 ==== 43425+0+0 (secure 0 0 0) 0x7ff66801c430 con 0x7ff6740fb0e0 2024-04-02T04:50:59.509 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.504+0000 7ff6717fa700 1 --2- 172.21.15.154:0/2750843789 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7ff664036880 0x7ff664038d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:50:59.509 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.504+0000 7ff67b1ac700 1 -- 172.21.15.154:0/2750843789 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff67405f5f0 con 0x7ff6740fb0e0 2024-04-02T04:50:59.509 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.504+0000 7ff6717fa700 1 -- 172.21.15.154:0/2750843789 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1151+0+0 (secure 0 0 0) 0x7ff668049460 con 0x7ff6740fb0e0 2024-04-02T04:50:59.509 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.504+0000 7ff673fff700 1 --2- 172.21.15.154:0/2750843789 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7ff664036880 0x7ff664038d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:50:59.509 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.505+0000 7ff673fff700 1 --2- 172.21.15.154:0/2750843789 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7ff664036880 0x7ff664038d30 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7ff660009990 tx=0x7ff660006e40 comp rx=0 tx=0).ready entity=mgr.14160 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:50:59.512 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.509+0000 7ff6717fa700 1 -- 172.21.15.154:0/2750843789 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7ff66801c7d0 con 0x7ff6740fb0e0 2024-04-02T04:50:59.770 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.767+0000 7ff67b1ac700 1 -- 172.21.15.154:0/2750843789 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7ff67404bed0 con 0x7ff6740fb0e0 2024-04-02T04:50:59.771 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.767+0000 7ff6717fa700 1 -- 172.21.15.154:0/2750843789 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+743 (secure 0 0 0) 0x7ff668021070 con 0x7ff6740fb0e0 2024-04-02T04:50:59.771 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:50:59.771 INFO:teuthology.orchestra.run.smithi154.stdout:{"epoch":1,"fsid":"2e0055d0-f0ac-11ee-b647-cb9ed24678a4","modified":"2024-04-02T04:48:56.938040Z","created":"2024-04-02T04:48:56.938040Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi152","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:3300","nonce":0},{"type":"v1","addr":"172.21.15.152:6789","nonce":0}]},"addr":"172.21.15.152:6789/0","public_addr":"172.21.15.152:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T04:50:59.773 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.769+0000 7ff67b1ac700 1 -- 172.21.15.154:0/2750843789 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7ff664036880 msgr2=0x7ff664038d30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:59.773 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.769+0000 7ff67b1ac700 1 --2- 172.21.15.154:0/2750843789 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7ff664036880 0x7ff664038d30 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7ff660009990 tx=0x7ff660006e40 comp rx=0 tx=0).stop 2024-04-02T04:50:59.773 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.769+0000 7ff67b1ac700 1 -- 172.21.15.154:0/2750843789 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff6740fb0e0 msgr2=0x7ff6740f77c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:50:59.773 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.769+0000 7ff67b1ac700 1 --2- 172.21.15.154:0/2750843789 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff6740fb0e0 0x7ff6740f77c0 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7ff668009680 tx=0x7ff668009e30 comp rx=0 tx=0).stop 2024-04-02T04:50:59.773 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.770+0000 7ff67b1ac700 1 -- 172.21.15.154:0/2750843789 shutdown_connections 2024-04-02T04:50:59.773 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.770+0000 7ff67b1ac700 1 --2- 172.21.15.154:0/2750843789 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7ff664036880 0x7ff664038d30 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:59.773 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.770+0000 7ff67b1ac700 1 --2- 172.21.15.154:0/2750843789 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff6740fb0e0 0x7ff6740f77c0 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:50:59.773 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.770+0000 7ff67b1ac700 1 -- 172.21.15.154:0/2750843789 >> 172.21.15.154:0/2750843789 conn(0x7ff6740f6840 msgr2=0x7ff6740f7d70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:50:59.773 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.770+0000 7ff67b1ac700 1 -- 172.21.15.154:0/2750843789 shutdown_connections 2024-04-02T04:50:59.774 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:50:59.770+0000 7ff67b1ac700 1 -- 172.21.15.154:0/2750843789 wait complete. 2024-04-02T04:50:59.774 INFO:teuthology.orchestra.run.smithi154.stderr:dumped monmap epoch 1 2024-04-02T04:51:00.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:50:59 smithi152 ceph-mon[29073]: from='client.? 172.21.15.154:0/2750843789' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:51:01.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:00 smithi152 ceph-mon[29073]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:01.289 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T04:51:01.289 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T04:51:01.540 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /etc/ceph/ceph.conf 2024-04-02T04:51:01.586 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /etc/ceph/ceph.conf 2024-04-02T04:51:03.056 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.051+0000 7f7b87d05700 1 -- 172.21.15.154:0/2148680304 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7b800fca80 msgr2=0x7f7b800fce90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:03.056 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.051+0000 7f7b87d05700 1 --2- 172.21.15.154:0/2148680304 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7b800fca80 0x7f7b800fce90 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f7b700048c0 tx=0x7f7b70009e30 comp rx=0 tx=0).stop 2024-04-02T04:51:03.056 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.052+0000 7f7b87d05700 1 -- 172.21.15.154:0/2148680304 shutdown_connections 2024-04-02T04:51:03.056 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.052+0000 7f7b87d05700 1 --2- 172.21.15.154:0/2148680304 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7b800fca80 0x7f7b800fce90 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:03.056 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.052+0000 7f7b87d05700 1 -- 172.21.15.154:0/2148680304 >> 172.21.15.154:0/2148680304 conn(0x7f7b800f81e0 msgr2=0x7f7b800fa610 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:51:03.056 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.052+0000 7f7b87d05700 1 -- 172.21.15.154:0/2148680304 shutdown_connections 2024-04-02T04:51:03.056 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.052+0000 7f7b87d05700 1 -- 172.21.15.154:0/2148680304 wait complete. 2024-04-02T04:51:03.056 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.052+0000 7f7b87d05700 1 Processor -- start 2024-04-02T04:51:03.056 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.052+0000 7f7b87d05700 1 -- start start 2024-04-02T04:51:03.056 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.052+0000 7f7b87d05700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7b800fca80 0x7f7b8010d0b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:51:03.056 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.052+0000 7f7b87d05700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7b8010d5f0 con 0x7f7b800fca80 2024-04-02T04:51:03.056 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.053+0000 7f7b85aa1700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7b800fca80 0x7f7b8010d0b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:51:03.057 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.053+0000 7f7b85aa1700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7b800fca80 0x7f7b8010d0b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.154:55864/0 (socket says 172.21.15.154:55864) 2024-04-02T04:51:03.057 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.053+0000 7f7b85aa1700 1 -- 172.21.15.154:0/233656300 learned_addr learned my addr 172.21.15.154:0/233656300 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-04-02T04:51:03.058 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.053+0000 7f7b85aa1700 1 -- 172.21.15.154:0/233656300 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7b70005040 con 0x7f7b800fca80 2024-04-02T04:51:03.058 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.053+0000 7f7b85aa1700 1 --2- 172.21.15.154:0/233656300 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7b800fca80 0x7f7b8010d0b0 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f7b700096b0 tx=0x7f7b70009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:51:03.058 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.053+0000 7f7b767fc700 1 -- 172.21.15.154:0/233656300 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7b7000bed0 con 0x7f7b800fca80 2024-04-02T04:51:03.058 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.053+0000 7f7b767fc700 1 -- 172.21.15.154:0/233656300 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f7b7000b5c0 con 0x7f7b800fca80 2024-04-02T04:51:03.058 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.053+0000 7f7b87d05700 1 -- 172.21.15.154:0/233656300 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7b8010d7f0 con 0x7f7b800fca80 2024-04-02T04:51:03.058 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.053+0000 7f7b767fc700 1 -- 172.21.15.154:0/233656300 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7b7001c5b0 con 0x7f7b800fca80 2024-04-02T04:51:03.058 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.053+0000 7f7b87d05700 1 -- 172.21.15.154:0/233656300 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7b8010e360 con 0x7f7b800fca80 2024-04-02T04:51:03.058 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.054+0000 7f7b87d05700 1 -- 172.21.15.154:0/233656300 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7b80102f90 con 0x7f7b800fca80 2024-04-02T04:51:03.058 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.054+0000 7f7b767fc700 1 -- 172.21.15.154:0/233656300 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 13) v1 ==== 43425+0+0 (secure 0 0 0) 0x7f7b70013420 con 0x7f7b800fca80 2024-04-02T04:51:03.059 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.054+0000 7f7b767fc700 1 --2- 172.21.15.154:0/233656300 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f7b6c036880 0x7f7b6c038d30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:51:03.059 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.054+0000 7f7b767fc700 1 -- 172.21.15.154:0/233656300 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1151+0+0 (secure 0 0 0) 0x7f7b70049ac0 con 0x7f7b800fca80 2024-04-02T04:51:03.059 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.054+0000 7f7b852a0700 1 --2- 172.21.15.154:0/233656300 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f7b6c036880 0x7f7b6c038d30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:51:03.059 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.055+0000 7f7b852a0700 1 --2- 172.21.15.154:0/233656300 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f7b6c036880 0x7f7b6c038d30 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7f7b7c009990 tx=0x7f7b7c006e40 comp rx=0 tx=0).ready entity=mgr.14160 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:51:03.062 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.059+0000 7f7b767fc700 1 -- 172.21.15.154:0/233656300 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f7b70013a80 con 0x7f7b800fca80 2024-04-02T04:51:03.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:02 smithi152 ceph-mon[29073]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:03.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:02 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:03.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:02 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:03.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:02 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:03.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:02 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-02T04:51:03.321 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.318+0000 7f7b87d05700 1 -- 172.21.15.154:0/233656300 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f7b8010ed70 con 0x7f7b800fca80 2024-04-02T04:51:03.322 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.319+0000 7f7b767fc700 1 -- 172.21.15.154:0/233656300 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+743 (secure 0 0 0) 0x7f7b70011370 con 0x7f7b800fca80 2024-04-02T04:51:03.322 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:51:03.322 INFO:teuthology.orchestra.run.smithi154.stdout:{"epoch":1,"fsid":"2e0055d0-f0ac-11ee-b647-cb9ed24678a4","modified":"2024-04-02T04:48:56.938040Z","created":"2024-04-02T04:48:56.938040Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi152","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:3300","nonce":0},{"type":"v1","addr":"172.21.15.152:6789","nonce":0}]},"addr":"172.21.15.152:6789/0","public_addr":"172.21.15.152:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T04:51:03.324 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.321+0000 7f7b87d05700 1 -- 172.21.15.154:0/233656300 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f7b6c036880 msgr2=0x7f7b6c038d30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:03.324 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.321+0000 7f7b87d05700 1 --2- 172.21.15.154:0/233656300 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f7b6c036880 0x7f7b6c038d30 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7f7b7c009990 tx=0x7f7b7c006e40 comp rx=0 tx=0).stop 2024-04-02T04:51:03.324 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.321+0000 7f7b87d05700 1 -- 172.21.15.154:0/233656300 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7b800fca80 msgr2=0x7f7b8010d0b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:03.324 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.321+0000 7f7b87d05700 1 --2- 172.21.15.154:0/233656300 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7b800fca80 0x7f7b8010d0b0 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7f7b700096b0 tx=0x7f7b70009e30 comp rx=0 tx=0).stop 2024-04-02T04:51:03.324 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.321+0000 7f7b87d05700 1 -- 172.21.15.154:0/233656300 shutdown_connections 2024-04-02T04:51:03.324 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.321+0000 7f7b87d05700 1 --2- 172.21.15.154:0/233656300 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7f7b6c036880 0x7f7b6c038d30 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:03.324 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.321+0000 7f7b87d05700 1 --2- 172.21.15.154:0/233656300 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f7b800fca80 0x7f7b8010d0b0 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:03.325 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.321+0000 7f7b87d05700 1 -- 172.21.15.154:0/233656300 >> 172.21.15.154:0/233656300 conn(0x7f7b800f81e0 msgr2=0x7f7b80103bc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:51:03.325 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.321+0000 7f7b87d05700 1 -- 172.21.15.154:0/233656300 shutdown_connections 2024-04-02T04:51:03.325 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:03.321+0000 7f7b87d05700 1 -- 172.21.15.154:0/233656300 wait complete. 2024-04-02T04:51:03.325 INFO:teuthology.orchestra.run.smithi154.stderr:dumped monmap epoch 1 2024-04-02T04:51:04.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:03 smithi152 ceph-mon[29073]: from='client.? 172.21.15.154:0/233656300' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:51:04.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:03 smithi152 ceph-mon[29073]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-02T04:51:04.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:03 smithi152 ceph-mon[29073]: mgrmap e14: smithi152.jivdcl(active, since 61s) 2024-04-02T04:51:04.997 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T04:51:04.997 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T04:51:05.250 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /etc/ceph/ceph.conf 2024-04-02T04:51:05.297 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /etc/ceph/ceph.conf 2024-04-02T04:51:06.615 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.610+0000 7fc1b8f2f700 1 -- 172.21.15.154:0/1462468221 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc1b40fb960 msgr2=0x7fc1b40fbd70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:06.616 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.610+0000 7fc1b8f2f700 1 --2- 172.21.15.154:0/1462468221 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc1b40fb960 0x7fc1b40fbd70 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7fc1ac0048c0 tx=0x7fc1ac009e30 comp rx=0 tx=0).stop 2024-04-02T04:51:06.616 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.610+0000 7fc1b8f2f700 1 -- 172.21.15.154:0/1462468221 shutdown_connections 2024-04-02T04:51:06.616 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.610+0000 7fc1b8f2f700 1 --2- 172.21.15.154:0/1462468221 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc1b40fb960 0x7fc1b40fbd70 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:06.616 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.610+0000 7fc1b8f2f700 1 -- 172.21.15.154:0/1462468221 >> 172.21.15.154:0/1462468221 conn(0x7fc1b40f70c0 msgr2=0x7fc1b40f94f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:51:06.616 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.610+0000 7fc1b8f2f700 1 -- 172.21.15.154:0/1462468221 shutdown_connections 2024-04-02T04:51:06.616 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.610+0000 7fc1b8f2f700 1 -- 172.21.15.154:0/1462468221 wait complete. 2024-04-02T04:51:06.616 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.611+0000 7fc1b8f2f700 1 Processor -- start 2024-04-02T04:51:06.616 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.611+0000 7fc1b8f2f700 1 -- start start 2024-04-02T04:51:06.616 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.611+0000 7fc1b8f2f700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc1b40fb960 0x7fc1b410bf90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:51:06.616 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.611+0000 7fc1b8f2f700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc1b410c4d0 con 0x7fc1b40fb960 2024-04-02T04:51:06.616 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.611+0000 7fc1b259c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc1b40fb960 0x7fc1b410bf90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:51:06.616 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.611+0000 7fc1b259c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc1b40fb960 0x7fc1b410bf90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.154:58698/0 (socket says 172.21.15.154:58698) 2024-04-02T04:51:06.616 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.611+0000 7fc1b259c700 1 -- 172.21.15.154:0/1755549450 learned_addr learned my addr 172.21.15.154:0/1755549450 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-04-02T04:51:06.616 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.611+0000 7fc1b259c700 1 -- 172.21.15.154:0/1755549450 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc1ac005040 con 0x7fc1b40fb960 2024-04-02T04:51:06.617 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.612+0000 7fc1b259c700 1 --2- 172.21.15.154:0/1755549450 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc1b40fb960 0x7fc1b410bf90 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7fc1ac0096b0 tx=0x7fc1ac009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:51:06.617 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.612+0000 7fc1aaffd700 1 -- 172.21.15.154:0/1755549450 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fc1ac00bed0 con 0x7fc1b40fb960 2024-04-02T04:51:06.617 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.612+0000 7fc1b8f2f700 1 -- 172.21.15.154:0/1755549450 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc1b410c6d0 con 0x7fc1b40fb960 2024-04-02T04:51:06.617 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.612+0000 7fc1aaffd700 1 -- 172.21.15.154:0/1755549450 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fc1ac00b5c0 con 0x7fc1b40fb960 2024-04-02T04:51:06.617 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.612+0000 7fc1aaffd700 1 -- 172.21.15.154:0/1755549450 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fc1ac01c5b0 con 0x7fc1b40fb960 2024-04-02T04:51:06.617 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.612+0000 7fc1b8f2f700 1 -- 172.21.15.154:0/1755549450 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc1b410d240 con 0x7fc1b40fb960 2024-04-02T04:51:06.618 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.613+0000 7fc1aaffd700 1 -- 172.21.15.154:0/1755549450 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 14) v1 ==== 43439+0+0 (secure 0 0 0) 0x7fc1ac01c820 con 0x7fc1b40fb960 2024-04-02T04:51:06.618 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.613+0000 7fc1b8f2f700 1 -- 172.21.15.154:0/1755549450 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc1b4101e70 con 0x7fc1b40fb960 2024-04-02T04:51:06.618 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.613+0000 7fc1aaffd700 1 --2- 172.21.15.154:0/1755549450 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fc1a00368d0 0x7fc1a0038d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:51:06.618 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.613+0000 7fc1aaffd700 1 -- 172.21.15.154:0/1755549450 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1151+0+0 (secure 0 0 0) 0x7fc1ac049b70 con 0x7fc1b40fb960 2024-04-02T04:51:06.618 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.613+0000 7fc1b1d9b700 1 -- 172.21.15.154:0/1755549450 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fc1a00368d0 msgr2=0x7fc1a0038d80 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.152:6800/2497421829 2024-04-02T04:51:06.618 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.613+0000 7fc1b1d9b700 1 --2- 172.21.15.154:0/1755549450 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fc1a00368d0 0x7fc1a0038d80 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-02T04:51:06.621 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.617+0000 7fc1aaffd700 1 -- 172.21.15.154:0/1755549450 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fc1ac0116f0 con 0x7fc1b40fb960 2024-04-02T04:51:06.817 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.814+0000 7fc1b1d9b700 1 -- 172.21.15.154:0/1755549450 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fc1a00368d0 msgr2=0x7fc1a0038d80 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.152:6800/2497421829 2024-04-02T04:51:06.817 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.814+0000 7fc1b1d9b700 1 --2- 172.21.15.154:0/1755549450 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fc1a00368d0 0x7fc1a0038d80 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.400000 2024-04-02T04:51:06.884 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.881+0000 7fc1b8f2f700 1 -- 172.21.15.154:0/1755549450 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fc1b410dc50 con 0x7fc1b40fb960 2024-04-02T04:51:06.885 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.882+0000 7fc1aaffd700 1 -- 172.21.15.154:0/1755549450 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+743 (secure 0 0 0) 0x7fc1ac024e20 con 0x7fc1b40fb960 2024-04-02T04:51:06.885 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:51:06.885 INFO:teuthology.orchestra.run.smithi154.stdout:{"epoch":1,"fsid":"2e0055d0-f0ac-11ee-b647-cb9ed24678a4","modified":"2024-04-02T04:48:56.938040Z","created":"2024-04-02T04:48:56.938040Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi152","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:3300","nonce":0},{"type":"v1","addr":"172.21.15.152:6789","nonce":0}]},"addr":"172.21.15.152:6789/0","public_addr":"172.21.15.152:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T04:51:06.887 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.884+0000 7fc1b8f2f700 1 -- 172.21.15.154:0/1755549450 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fc1a00368d0 msgr2=0x7fc1a0038d80 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-02T04:51:06.887 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.884+0000 7fc1b8f2f700 1 --2- 172.21.15.154:0/1755549450 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fc1a00368d0 0x7fc1a0038d80 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:06.887 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.884+0000 7fc1b8f2f700 1 -- 172.21.15.154:0/1755549450 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc1b40fb960 msgr2=0x7fc1b410bf90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:06.887 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.884+0000 7fc1b8f2f700 1 --2- 172.21.15.154:0/1755549450 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc1b40fb960 0x7fc1b410bf90 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7fc1ac0096b0 tx=0x7fc1ac009e30 comp rx=0 tx=0).stop 2024-04-02T04:51:06.888 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.884+0000 7fc1b8f2f700 1 -- 172.21.15.154:0/1755549450 shutdown_connections 2024-04-02T04:51:06.888 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.884+0000 7fc1b8f2f700 1 --2- 172.21.15.154:0/1755549450 >> [v2:172.21.15.152:6800/2497421829,v1:172.21.15.152:6801/2497421829] conn(0x7fc1a00368d0 0x7fc1a0038d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:06.888 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.884+0000 7fc1b8f2f700 1 --2- 172.21.15.154:0/1755549450 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc1b40fb960 0x7fc1b410bf90 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:06.888 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.884+0000 7fc1b8f2f700 1 -- 172.21.15.154:0/1755549450 >> 172.21.15.154:0/1755549450 conn(0x7fc1b40f70c0 msgr2=0x7fc1b4102aa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:51:06.888 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.884+0000 7fc1b8f2f700 1 -- 172.21.15.154:0/1755549450 shutdown_connections 2024-04-02T04:51:06.888 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:06.884+0000 7fc1b8f2f700 1 -- 172.21.15.154:0/1755549450 wait complete. 2024-04-02T04:51:06.888 INFO:teuthology.orchestra.run.smithi154.stderr:dumped monmap epoch 1 2024-04-02T04:51:07.006 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:06 smithi152 ceph-mon[29073]: from='client.? 172.21.15.154:0/1755549450' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:51:08.458 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T04:51:08.458 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T04:51:08.703 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /etc/ceph/ceph.conf 2024-04-02T04:51:08.750 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /etc/ceph/ceph.conf 2024-04-02T04:51:10.319 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.314+0000 7f2d2333b700 1 -- 172.21.15.154:0/3386761603 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2d1c0fbf60 msgr2=0x7f2d1c0fc370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:10.319 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.314+0000 7f2d2333b700 1 --2- 172.21.15.154:0/3386761603 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2d1c0fbf60 0x7f2d1c0fc370 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f2d180048c0 tx=0x7f2d18009e30 comp rx=0 tx=0).stop 2024-04-02T04:51:10.319 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.314+0000 7f2d2333b700 1 -- 172.21.15.154:0/3386761603 shutdown_connections 2024-04-02T04:51:10.319 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.314+0000 7f2d2333b700 1 --2- 172.21.15.154:0/3386761603 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2d1c0fbf60 0x7f2d1c0fc370 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:10.320 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.314+0000 7f2d2333b700 1 -- 172.21.15.154:0/3386761603 >> 172.21.15.154:0/3386761603 conn(0x7f2d1c0f6850 msgr2=0x7f2d1c0f8c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:51:10.320 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.314+0000 7f2d2333b700 1 -- 172.21.15.154:0/3386761603 shutdown_connections 2024-04-02T04:51:10.320 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.315+0000 7f2d2333b700 1 -- 172.21.15.154:0/3386761603 wait complete. 2024-04-02T04:51:10.320 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.315+0000 7f2d2333b700 1 Processor -- start 2024-04-02T04:51:10.320 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.315+0000 7f2d2333b700 1 -- start start 2024-04-02T04:51:10.320 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.315+0000 7f2d2333b700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2d1c0fbf60 0x7f2d1c0fb760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:51:10.320 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.315+0000 7f2d2333b700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2d1c10fde0 con 0x7f2d1c0fbf60 2024-04-02T04:51:10.320 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.315+0000 7f2d210d7700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2d1c0fbf60 0x7f2d1c0fb760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:51:10.320 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.315+0000 7f2d210d7700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2d1c0fbf60 0x7f2d1c0fb760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.154:58722/0 (socket says 172.21.15.154:58722) 2024-04-02T04:51:10.320 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.315+0000 7f2d210d7700 1 -- 172.21.15.154:0/2454352752 learned_addr learned my addr 172.21.15.154:0/2454352752 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-04-02T04:51:10.320 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.316+0000 7f2d210d7700 1 -- 172.21.15.154:0/2454352752 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2d18005040 con 0x7f2d1c0fbf60 2024-04-02T04:51:10.320 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.316+0000 7f2d210d7700 1 --2- 172.21.15.154:0/2454352752 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2d1c0fbf60 0x7f2d1c0fb760 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f2d18009680 tx=0x7f2d18009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:51:10.320 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.316+0000 7f2d11ffb700 1 -- 172.21.15.154:0/2454352752 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f2d1800bed0 con 0x7f2d1c0fbf60 2024-04-02T04:51:10.320 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.316+0000 7f2d2333b700 1 -- 172.21.15.154:0/2454352752 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2d1c10ff20 con 0x7f2d1c0fbf60 2024-04-02T04:51:10.320 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.316+0000 7f2d11ffb700 1 -- 172.21.15.154:0/2454352752 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f2d1800b5c0 con 0x7f2d1c0fbf60 2024-04-02T04:51:10.321 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.316+0000 7f2d11ffb700 1 -- 172.21.15.154:0/2454352752 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f2d1801c5b0 con 0x7f2d1c0fbf60 2024-04-02T04:51:10.321 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.316+0000 7f2d2333b700 1 -- 172.21.15.154:0/2454352752 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2d1c110410 con 0x7f2d1c0fbf60 2024-04-02T04:51:10.321 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.317+0000 7f2d11ffb700 1 -- 172.21.15.154:0/2454352752 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 15) v1 ==== 43162+0+0 (secure 0 0 0) 0x7f2d1801c710 con 0x7f2d1c0fbf60 2024-04-02T04:51:10.321 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.317+0000 7f2d11ffb700 1 -- 172.21.15.154:0/2454352752 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1366+0+0 (secure 0 0 0) 0x7f2d18049840 con 0x7f2d1c0fbf60 2024-04-02T04:51:10.321 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.317+0000 7f2d2333b700 1 -- 172.21.15.154:0/2454352752 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2d1c105920 con 0x7f2d1c0fbf60 2024-04-02T04:51:10.330 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.327+0000 7f2d11ffb700 1 -- 172.21.15.154:0/2454352752 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f2d18025400 con 0x7f2d1c0fbf60 2024-04-02T04:51:10.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:10 smithi152 ceph-mon[29073]: Active manager daemon smithi152.jivdcl restarted 2024-04-02T04:51:10.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:10 smithi152 ceph-mon[29073]: Activating manager daemon smithi152.jivdcl 2024-04-02T04:51:10.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:10 smithi152 ceph-mon[29073]: osdmap e5: 0 total, 0 up, 0 in 2024-04-02T04:51:10.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:10 smithi152 ceph-mon[29073]: mgrmap e15: smithi152.jivdcl(active, starting, since 0.0033924s) 2024-04-02T04:51:10.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:10 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi152"}]: dispatch 2024-04-02T04:51:10.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:10 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mgr metadata", "who": "smithi152.jivdcl", "id": "smithi152.jivdcl"}]: dispatch 2024-04-02T04:51:10.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:10 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T04:51:10.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:10 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T04:51:10.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:10 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T04:51:10.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:10 smithi152 ceph-mon[29073]: Manager daemon smithi152.jivdcl is now available 2024-04-02T04:51:10.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:10 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:51:10.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:10 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:51:10.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:10 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:10.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:10 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:51:10.632 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.629+0000 7f2d2333b700 1 -- 172.21.15.154:0/2454352752 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f2d1c101740 con 0x7f2d1c0fbf60 2024-04-02T04:51:10.633 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.629+0000 7f2d11ffb700 1 -- 172.21.15.154:0/2454352752 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+743 (secure 0 0 0) 0x7f2d18011b60 con 0x7f2d1c0fbf60 2024-04-02T04:51:10.633 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:51:10.633 INFO:teuthology.orchestra.run.smithi154.stdout:{"epoch":1,"fsid":"2e0055d0-f0ac-11ee-b647-cb9ed24678a4","modified":"2024-04-02T04:48:56.938040Z","created":"2024-04-02T04:48:56.938040Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi152","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:3300","nonce":0},{"type":"v1","addr":"172.21.15.152:6789","nonce":0}]},"addr":"172.21.15.152:6789/0","public_addr":"172.21.15.152:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T04:51:10.636 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.632+0000 7f2d2333b700 1 -- 172.21.15.154:0/2454352752 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2d1c0fbf60 msgr2=0x7f2d1c0fb760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:10.636 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.633+0000 7f2d2333b700 1 --2- 172.21.15.154:0/2454352752 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2d1c0fbf60 0x7f2d1c0fb760 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f2d18009680 tx=0x7f2d18009e30 comp rx=0 tx=0).stop 2024-04-02T04:51:10.636 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.633+0000 7f2d2333b700 1 -- 172.21.15.154:0/2454352752 shutdown_connections 2024-04-02T04:51:10.636 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.633+0000 7f2d2333b700 1 --2- 172.21.15.154:0/2454352752 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2d1c0fbf60 0x7f2d1c0fb760 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:10.636 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.633+0000 7f2d2333b700 1 -- 172.21.15.154:0/2454352752 >> 172.21.15.154:0/2454352752 conn(0x7f2d1c0f6850 msgr2=0x7f2d1c102aa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:51:10.636 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.633+0000 7f2d2333b700 1 -- 172.21.15.154:0/2454352752 shutdown_connections 2024-04-02T04:51:10.636 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:10.633+0000 7f2d2333b700 1 -- 172.21.15.154:0/2454352752 wait complete. 2024-04-02T04:51:10.637 INFO:teuthology.orchestra.run.smithi154.stderr:dumped monmap epoch 1 2024-04-02T04:51:11.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:11 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi152.jivdcl/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:51:11.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:11 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi152.jivdcl/trash_purge_schedule"}]: dispatch 2024-04-02T04:51:11.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:11 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:11.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:11 smithi152 ceph-mon[29073]: from='client.? 172.21.15.154:0/2454352752' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:51:11.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:11 smithi152 ceph-mon[29073]: mgrmap e16: smithi152.jivdcl(active, since 1.00831s) 2024-04-02T04:51:12.338 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T04:51:12.339 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T04:51:12.362 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:12 smithi152 ceph-mon[29073]: [02/Apr/2024:04:51:10] ENGINE Bus STARTING 2024-04-02T04:51:12.362 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:12 smithi152 ceph-mon[29073]: [02/Apr/2024:04:51:11] ENGINE Serving on https://172.21.15.152:7150 2024-04-02T04:51:12.362 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:12 smithi152 ceph-mon[29073]: [02/Apr/2024:04:51:11] ENGINE Bus STARTED 2024-04-02T04:51:12.362 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:12 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:12.362 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:12 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:13.230 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /etc/ceph/ceph.conf 2024-04-02T04:51:13.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:13 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:13.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:13 smithi152 ceph-mon[29073]: mgrmap e17: smithi152.jivdcl(active, since 2s) 2024-04-02T04:51:13.642 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /etc/ceph/ceph.conf 2024-04-02T04:51:15.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:14 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:15.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:14 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:15.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:14 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:15.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:14 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:15.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:14 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd/host:smithi152", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:51:15.948 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.935+0000 7f61a5b2f700 1 -- 172.21.15.154:0/1774812680 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61a0066a00 msgr2=0x7f61a00ff650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:15.948 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.935+0000 7f61a5b2f700 1 --2- 172.21.15.154:0/1774812680 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61a0066a00 0x7f61a00ff650 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7f61900048c0 tx=0x7f6190009e30 comp rx=0 tx=0).stop 2024-04-02T04:51:15.948 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.935+0000 7f61a5b2f700 1 -- 172.21.15.154:0/1774812680 shutdown_connections 2024-04-02T04:51:15.948 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.935+0000 7f61a5b2f700 1 --2- 172.21.15.154:0/1774812680 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61a0066a00 0x7f61a00ff650 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:15.948 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.935+0000 7f61a5b2f700 1 -- 172.21.15.154:0/1774812680 >> 172.21.15.154:0/1774812680 conn(0x7f61a00f70c0 msgr2=0x7f61a00f94f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:51:15.949 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.935+0000 7f61a5b2f700 1 -- 172.21.15.154:0/1774812680 shutdown_connections 2024-04-02T04:51:15.949 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.935+0000 7f61a5b2f700 1 -- 172.21.15.154:0/1774812680 wait complete. 2024-04-02T04:51:15.949 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.935+0000 7f61a5b2f700 1 Processor -- start 2024-04-02T04:51:15.949 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.935+0000 7f61a5b2f700 1 -- start start 2024-04-02T04:51:15.949 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.935+0000 7f61a5b2f700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61a0066a00 0x7f61a00fcfa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:51:15.949 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.935+0000 7f61a5b2f700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f61a00fd4e0 con 0x7f61a0066a00 2024-04-02T04:51:15.949 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.936+0000 7f619f7fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61a0066a00 0x7f61a00fcfa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:51:15.949 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.936+0000 7f619f7fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61a0066a00 0x7f61a00fcfa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.154:37270/0 (socket says 172.21.15.154:37270) 2024-04-02T04:51:15.949 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.936+0000 7f619f7fe700 1 -- 172.21.15.154:0/229313689 learned_addr learned my addr 172.21.15.154:0/229313689 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-04-02T04:51:15.949 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.936+0000 7f619f7fe700 1 -- 172.21.15.154:0/229313689 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6190005040 con 0x7f61a0066a00 2024-04-02T04:51:15.949 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.936+0000 7f619f7fe700 1 --2- 172.21.15.154:0/229313689 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61a0066a00 0x7f61a00fcfa0 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f619000b3b0 tx=0x7f6190009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:51:15.949 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.936+0000 7f6187fff700 1 -- 172.21.15.154:0/229313689 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6190004e60 con 0x7f61a0066a00 2024-04-02T04:51:15.950 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.936+0000 7f61a5b2f700 1 -- 172.21.15.154:0/229313689 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f61a00fb5b0 con 0x7f61a0066a00 2024-04-02T04:51:15.950 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.936+0000 7f6187fff700 1 -- 172.21.15.154:0/229313689 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f6190009650 con 0x7f61a0066a00 2024-04-02T04:51:15.950 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.936+0000 7f6187fff700 1 -- 172.21.15.154:0/229313689 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f619001c5b0 con 0x7f61a0066a00 2024-04-02T04:51:15.950 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.936+0000 7f61a5b2f700 1 -- 172.21.15.154:0/229313689 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f61a00fbaa0 con 0x7f61a0066a00 2024-04-02T04:51:15.950 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.937+0000 7f61a5b2f700 1 -- 172.21.15.154:0/229313689 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f61a005f5f0 con 0x7f61a0066a00 2024-04-02T04:51:15.950 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.937+0000 7f6187fff700 1 -- 172.21.15.154:0/229313689 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 17) v1 ==== 43483+0+0 (secure 0 0 0) 0x7f619001c870 con 0x7f61a0066a00 2024-04-02T04:51:15.950 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.937+0000 7f6187fff700 1 --2- 172.21.15.154:0/229313689 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f6188036970 0x7f6188038e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:51:15.950 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.937+0000 7f6187fff700 1 -- 172.21.15.154:0/229313689 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1366+0+0 (secure 0 0 0) 0x7f6190049c90 con 0x7f61a0066a00 2024-04-02T04:51:15.950 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.938+0000 7f619effd700 1 --2- 172.21.15.154:0/229313689 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f6188036970 0x7f6188038e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:51:15.950 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.938+0000 7f619effd700 1 --2- 172.21.15.154:0/229313689 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f6188036970 0x7f6188038e20 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7f6194009990 tx=0x7f6194006e40 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:51:15.950 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:15.942+0000 7f6187fff700 1 -- 172.21.15.154:0/229313689 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f6190023300 con 0x7f61a0066a00 2024-04-02T04:51:16.219 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:16.215+0000 7f61a5b2f700 1 -- 172.21.15.154:0/229313689 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f61a004bed0 con 0x7f61a0066a00 2024-04-02T04:51:16.276 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:16.215+0000 7f6187fff700 1 -- 172.21.15.154:0/229313689 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+743 (secure 0 0 0) 0x7f6190011b60 con 0x7f61a0066a00 2024-04-02T04:51:16.277 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:51:16.277 INFO:teuthology.orchestra.run.smithi154.stdout:{"epoch":1,"fsid":"2e0055d0-f0ac-11ee-b647-cb9ed24678a4","modified":"2024-04-02T04:48:56.938040Z","created":"2024-04-02T04:48:56.938040Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi152","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:3300","nonce":0},{"type":"v1","addr":"172.21.15.152:6789","nonce":0}]},"addr":"172.21.15.152:6789/0","public_addr":"172.21.15.152:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T04:51:16.277 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:16.218+0000 7f61a5b2f700 1 -- 172.21.15.154:0/229313689 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f6188036970 msgr2=0x7f6188038e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:16.277 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:16.218+0000 7f61a5b2f700 1 --2- 172.21.15.154:0/229313689 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f6188036970 0x7f6188038e20 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0x7f6194009990 tx=0x7f6194006e40 comp rx=0 tx=0).stop 2024-04-02T04:51:16.278 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:16.218+0000 7f61a5b2f700 1 -- 172.21.15.154:0/229313689 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61a0066a00 msgr2=0x7f61a00fcfa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:16.278 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:16.218+0000 7f61a5b2f700 1 --2- 172.21.15.154:0/229313689 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61a0066a00 0x7f61a00fcfa0 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7f619000b3b0 tx=0x7f6190009e30 comp rx=0 tx=0).stop 2024-04-02T04:51:16.278 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:16.218+0000 7f61a5b2f700 1 -- 172.21.15.154:0/229313689 shutdown_connections 2024-04-02T04:51:16.278 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:16.218+0000 7f61a5b2f700 1 --2- 172.21.15.154:0/229313689 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f6188036970 0x7f6188038e20 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:16.278 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:16.218+0000 7f61a5b2f700 1 --2- 172.21.15.154:0/229313689 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61a0066a00 0x7f61a00fcfa0 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:16.278 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:16.218+0000 7f61a5b2f700 1 -- 172.21.15.154:0/229313689 >> 172.21.15.154:0/229313689 conn(0x7f61a00f70c0 msgr2=0x7f61a0103b90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:51:16.278 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:16.218+0000 7f61a5b2f700 1 -- 172.21.15.154:0/229313689 shutdown_connections 2024-04-02T04:51:16.278 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:16.218+0000 7f61a5b2f700 1 -- 172.21.15.154:0/229313689 wait complete. 2024-04-02T04:51:16.278 INFO:teuthology.orchestra.run.smithi154.stderr:dumped monmap epoch 1 2024-04-02T04:51:16.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:16 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:16.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:16 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:16.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:16 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:16.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:16 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:16.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:16 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd/host:smithi154", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:51:16.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:16 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:51:16.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:16 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:51:16.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:16 smithi152 ceph-mon[29073]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-02T04:51:16.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:16 smithi152 ceph-mon[29073]: Updating smithi154:/etc/ceph/ceph.conf 2024-04-02T04:51:16.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:16 smithi152 ceph-mon[29073]: Updating smithi152:/var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T04:51:16.508 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:16 smithi152 ceph-mon[29073]: Updating smithi154:/var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T04:51:16.508 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:16 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:16.508 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:16 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:17.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:17 smithi152 ceph-mon[29073]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:51:17.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:17 smithi152 ceph-mon[29073]: Updating smithi154:/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:51:17.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:17 smithi152 ceph-mon[29073]: Updating smithi152:/var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T04:51:17.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:17 smithi152 ceph-mon[29073]: Updating smithi154:/var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T04:51:17.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:17 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:17.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:17 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:17.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:17 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:17.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:17 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi154", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-02T04:51:17.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:17 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi154", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-02T04:51:17.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:17 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:51:17.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:17 smithi152 ceph-mon[29073]: Deploying daemon crash.smithi154 on smithi154 2024-04-02T04:51:17.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:17 smithi152 ceph-mon[29073]: from='client.? 172.21.15.154:0/229313689' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:51:17.968 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T04:51:17.968 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T04:51:18.258 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T04:51:20.131 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.123+0000 7f544cbd1700 1 -- 172.21.15.154:0/3087005546 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f5448102ac0 msgr2=0x7f5448102e90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:20.131 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.123+0000 7f544cbd1700 1 --2- 172.21.15.154:0/3087005546 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f5448102ac0 0x7f5448102e90 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7f54380048c0 tx=0x7f5438009e30 comp rx=0 tx=0).stop 2024-04-02T04:51:20.131 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.123+0000 7f544cbd1700 1 -- 172.21.15.154:0/3087005546 shutdown_connections 2024-04-02T04:51:20.131 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.123+0000 7f544cbd1700 1 --2- 172.21.15.154:0/3087005546 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f5448102ac0 0x7f5448102e90 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:20.131 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.123+0000 7f544cbd1700 1 -- 172.21.15.154:0/3087005546 >> 172.21.15.154:0/3087005546 conn(0x7f54480f85e0 msgr2=0x7f54480fa9f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:51:20.131 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.123+0000 7f544cbd1700 1 -- 172.21.15.154:0/3087005546 shutdown_connections 2024-04-02T04:51:20.131 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.123+0000 7f544cbd1700 1 -- 172.21.15.154:0/3087005546 wait complete. 2024-04-02T04:51:20.131 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.124+0000 7f544cbd1700 1 Processor -- start 2024-04-02T04:51:20.131 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.124+0000 7f544cbd1700 1 -- start start 2024-04-02T04:51:20.131 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.124+0000 7f544cbd1700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f5448102ac0 0x7f5448113120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:51:20.131 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.124+0000 7f544cbd1700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5448113660 con 0x7f5448102ac0 2024-04-02T04:51:20.132 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.124+0000 7f544659c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f5448102ac0 0x7f5448113120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:51:20.132 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.124+0000 7f544659c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f5448102ac0 0x7f5448113120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.154:37306/0 (socket says 172.21.15.154:37306) 2024-04-02T04:51:20.132 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.124+0000 7f544659c700 1 -- 172.21.15.154:0/1791533700 learned_addr learned my addr 172.21.15.154:0/1791533700 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-04-02T04:51:20.132 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.124+0000 7f544659c700 1 -- 172.21.15.154:0/1791533700 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5438005040 con 0x7f5448102ac0 2024-04-02T04:51:20.132 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.125+0000 7f544659c700 1 --2- 172.21.15.154:0/1791533700 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f5448102ac0 0x7f5448113120 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f543800b820 tx=0x7f5438009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:51:20.132 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.125+0000 7f5436ffd700 1 -- 172.21.15.154:0/1791533700 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f54380093c0 con 0x7f5448102ac0 2024-04-02T04:51:20.132 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.125+0000 7f544cbd1700 1 -- 172.21.15.154:0/1791533700 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5448113860 con 0x7f5448102ac0 2024-04-02T04:51:20.132 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.125+0000 7f5436ffd700 1 -- 172.21.15.154:0/1791533700 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f5438013420 con 0x7f5448102ac0 2024-04-02T04:51:20.132 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.125+0000 7f5436ffd700 1 -- 172.21.15.154:0/1791533700 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f543801c5b0 con 0x7f5448102ac0 2024-04-02T04:51:20.132 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.125+0000 7f544cbd1700 1 -- 172.21.15.154:0/1791533700 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5448113d50 con 0x7f5448102ac0 2024-04-02T04:51:20.132 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.126+0000 7f5436ffd700 1 -- 172.21.15.154:0/1791533700 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 17) v1 ==== 43483+0+0 (secure 0 0 0) 0x7f543801c7f0 con 0x7f5448102ac0 2024-04-02T04:51:20.132 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.126+0000 7f544cbd1700 1 -- 172.21.15.154:0/1791533700 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f54480fe1c0 con 0x7f5448102ac0 2024-04-02T04:51:20.132 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.126+0000 7f5436ffd700 1 --2- 172.21.15.154:0/1791533700 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f5430036970 0x7f5430038e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:51:20.132 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.126+0000 7f5436ffd700 1 -- 172.21.15.154:0/1791533700 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1366+0+0 (secure 0 0 0) 0x7f5438049a70 con 0x7f5448102ac0 2024-04-02T04:51:20.132 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.126+0000 7f5445d9b700 1 --2- 172.21.15.154:0/1791533700 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f5430036970 0x7f5430038e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:51:20.133 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.127+0000 7f5445d9b700 1 --2- 172.21.15.154:0/1791533700 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f5430036970 0x7f5430038e20 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f543c009990 tx=0x7f543c006e40 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:51:20.135 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.131+0000 7f5436ffd700 1 -- 172.21.15.154:0/1791533700 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f5438023300 con 0x7f5448102ac0 2024-04-02T04:51:20.428 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.423+0000 7f544cbd1700 1 -- 172.21.15.154:0/1791533700 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f544805f5f0 con 0x7f5448102ac0 2024-04-02T04:51:20.429 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.424+0000 7f5436ffd700 1 -- 172.21.15.154:0/1791533700 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+743 (secure 0 0 0) 0x7f5438011b60 con 0x7f5448102ac0 2024-04-02T04:51:20.429 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:51:20.429 INFO:teuthology.orchestra.run.smithi154.stdout:{"epoch":1,"fsid":"2e0055d0-f0ac-11ee-b647-cb9ed24678a4","modified":"2024-04-02T04:48:56.938040Z","created":"2024-04-02T04:48:56.938040Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi152","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:3300","nonce":0},{"type":"v1","addr":"172.21.15.152:6789","nonce":0}]},"addr":"172.21.15.152:6789/0","public_addr":"172.21.15.152:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T04:51:20.430 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.426+0000 7f544cbd1700 1 -- 172.21.15.154:0/1791533700 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f5430036970 msgr2=0x7f5430038e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:20.431 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.426+0000 7f544cbd1700 1 --2- 172.21.15.154:0/1791533700 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f5430036970 0x7f5430038e20 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0x7f543c009990 tx=0x7f543c006e40 comp rx=0 tx=0).stop 2024-04-02T04:51:20.431 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.426+0000 7f544cbd1700 1 -- 172.21.15.154:0/1791533700 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f5448102ac0 msgr2=0x7f5448113120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:20.431 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.426+0000 7f544cbd1700 1 --2- 172.21.15.154:0/1791533700 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f5448102ac0 0x7f5448113120 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7f543800b820 tx=0x7f5438009e30 comp rx=0 tx=0).stop 2024-04-02T04:51:20.431 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.426+0000 7f544cbd1700 1 -- 172.21.15.154:0/1791533700 shutdown_connections 2024-04-02T04:51:20.431 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.426+0000 7f544cbd1700 1 --2- 172.21.15.154:0/1791533700 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f5430036970 0x7f5430038e20 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:20.431 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.426+0000 7f544cbd1700 1 --2- 172.21.15.154:0/1791533700 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f5448102ac0 0x7f5448113120 unknown :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:20.431 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.426+0000 7f544cbd1700 1 -- 172.21.15.154:0/1791533700 >> 172.21.15.154:0/1791533700 conn(0x7f54480f85e0 msgr2=0x7f54480f9bd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:51:20.431 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.427+0000 7f544cbd1700 1 -- 172.21.15.154:0/1791533700 shutdown_connections 2024-04-02T04:51:20.432 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:20.427+0000 7f544cbd1700 1 -- 172.21.15.154:0/1791533700 wait complete. 2024-04-02T04:51:20.432 INFO:teuthology.orchestra.run.smithi154.stderr:dumped monmap epoch 1 2024-04-02T04:51:21.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:20 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:21.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:20 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:21.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:20 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:21.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:20 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:21.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:20 smithi152 ceph-mon[29073]: Deploying daemon node-exporter.smithi154 on smithi154 2024-04-02T04:51:21.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:20 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:21.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:20 smithi152 ceph-mon[29073]: from='client.? 172.21.15.154:0/1791533700' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:51:22.042 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T04:51:22.043 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T04:51:22.941 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T04:51:25.709 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.705+0000 7f59c22e2700 1 -- 172.21.15.154:0/655626962 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f59bc1019b0 msgr2=0x7f59bc101d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:25.710 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.705+0000 7f59c22e2700 1 --2- 172.21.15.154:0/655626962 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f59bc1019b0 0x7f59bc101d80 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7f59b00048c0 tx=0x7f59b0009e30 comp rx=0 tx=0).stop 2024-04-02T04:51:25.710 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.705+0000 7f59c22e2700 1 -- 172.21.15.154:0/655626962 shutdown_connections 2024-04-02T04:51:25.710 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.705+0000 7f59c22e2700 1 --2- 172.21.15.154:0/655626962 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f59bc1019b0 0x7f59bc101d80 unknown :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:25.710 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.705+0000 7f59c22e2700 1 -- 172.21.15.154:0/655626962 >> 172.21.15.154:0/655626962 conn(0x7f59bc0f7510 msgr2=0x7f59bc0f9920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:51:25.710 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.705+0000 7f59c22e2700 1 -- 172.21.15.154:0/655626962 shutdown_connections 2024-04-02T04:51:25.710 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.706+0000 7f59c22e2700 1 -- 172.21.15.154:0/655626962 wait complete. 2024-04-02T04:51:25.710 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.706+0000 7f59c22e2700 1 Processor -- start 2024-04-02T04:51:25.710 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.706+0000 7f59c22e2700 1 -- start start 2024-04-02T04:51:25.710 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.706+0000 7f59c22e2700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f59bc1019b0 0x7f59bc0f79b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:51:25.710 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.706+0000 7f59c22e2700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f59bc0f7ef0 con 0x7f59bc1019b0 2024-04-02T04:51:25.711 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.707+0000 7f59bbfff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f59bc1019b0 0x7f59bc0f79b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:51:25.711 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.707+0000 7f59bbfff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f59bc1019b0 0x7f59bc0f79b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.154:36250/0 (socket says 172.21.15.154:36250) 2024-04-02T04:51:25.711 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.707+0000 7f59bbfff700 1 -- 172.21.15.154:0/656502052 learned_addr learned my addr 172.21.15.154:0/656502052 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-04-02T04:51:25.711 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.707+0000 7f59bbfff700 1 -- 172.21.15.154:0/656502052 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f59b0005040 con 0x7f59bc1019b0 2024-04-02T04:51:25.712 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.707+0000 7f59bbfff700 1 --2- 172.21.15.154:0/656502052 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f59bc1019b0 0x7f59bc0f79b0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f59b000b820 tx=0x7f59b0009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:51:25.712 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.707+0000 7f59b8ff9700 1 -- 172.21.15.154:0/656502052 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f59b00093c0 con 0x7f59bc1019b0 2024-04-02T04:51:25.712 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.707+0000 7f59c22e2700 1 -- 172.21.15.154:0/656502052 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f59bc0f80f0 con 0x7f59bc1019b0 2024-04-02T04:51:25.712 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.707+0000 7f59b8ff9700 1 -- 172.21.15.154:0/656502052 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f59b0013420 con 0x7f59bc1019b0 2024-04-02T04:51:25.712 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.707+0000 7f59b8ff9700 1 -- 172.21.15.154:0/656502052 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f59b001c5b0 con 0x7f59bc1019b0 2024-04-02T04:51:25.712 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.708+0000 7f59c22e2700 1 -- 172.21.15.154:0/656502052 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f59bc0f85e0 con 0x7f59bc1019b0 2024-04-02T04:51:25.712 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.708+0000 7f59c22e2700 1 -- 172.21.15.154:0/656502052 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f59bc0f8820 con 0x7f59bc1019b0 2024-04-02T04:51:25.712 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.708+0000 7f59b8ff9700 1 -- 172.21.15.154:0/656502052 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 17) v1 ==== 43483+0+0 (secure 0 0 0) 0x7f59b0013a90 con 0x7f59bc1019b0 2024-04-02T04:51:25.713 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.709+0000 7f59b8ff9700 1 --2- 172.21.15.154:0/656502052 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f59a4036970 0x7f59a4038e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:51:25.713 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.709+0000 7f59b8ff9700 1 -- 172.21.15.154:0/656502052 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1366+0+0 (secure 0 0 0) 0x7f59b0049a90 con 0x7f59bc1019b0 2024-04-02T04:51:25.713 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.709+0000 7f59bb7fe700 1 --2- 172.21.15.154:0/656502052 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f59a4036970 0x7f59a4038e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:51:25.713 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.709+0000 7f59bb7fe700 1 --2- 172.21.15.154:0/656502052 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f59a4036970 0x7f59a4038e20 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f59ac009990 tx=0x7f59ac006e40 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:51:25.717 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.714+0000 7f59b8ff9700 1 -- 172.21.15.154:0/656502052 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f59bc0f8820 con 0x7f59bc1019b0 2024-04-02T04:51:25.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:25 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:25.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:25 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:25.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:25 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:25.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:25 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:25.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:25 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi154.pplnxz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T04:51:25.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:25 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi154.pplnxz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-02T04:51:25.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:25 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T04:51:25.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:25 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:51:25.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:25 smithi152 ceph-mon[29073]: Deploying daemon mgr.smithi154.pplnxz on smithi154 2024-04-02T04:51:25.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:25 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:25.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:25 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:51:25.984 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.980+0000 7f59c22e2700 1 -- 172.21.15.154:0/656502052 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f59bc0f8820 con 0x7f59bc1019b0 2024-04-02T04:51:25.984 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.981+0000 7f59b8ff9700 1 -- 172.21.15.154:0/656502052 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+743 (secure 0 0 0) 0x7f59bc0f8820 con 0x7f59bc1019b0 2024-04-02T04:51:25.984 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:51:25.985 INFO:teuthology.orchestra.run.smithi154.stdout:{"epoch":1,"fsid":"2e0055d0-f0ac-11ee-b647-cb9ed24678a4","modified":"2024-04-02T04:48:56.938040Z","created":"2024-04-02T04:48:56.938040Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi152","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:3300","nonce":0},{"type":"v1","addr":"172.21.15.152:6789","nonce":0}]},"addr":"172.21.15.152:6789/0","public_addr":"172.21.15.152:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T04:51:25.986 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.983+0000 7f59c22e2700 1 -- 172.21.15.154:0/656502052 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f59a4036970 msgr2=0x7f59a4038e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:25.987 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.983+0000 7f59c22e2700 1 --2- 172.21.15.154:0/656502052 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f59a4036970 0x7f59a4038e20 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0x7f59ac009990 tx=0x7f59ac006e40 comp rx=0 tx=0).stop 2024-04-02T04:51:25.987 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.983+0000 7f59c22e2700 1 -- 172.21.15.154:0/656502052 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f59bc1019b0 msgr2=0x7f59bc0f79b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:25.987 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.983+0000 7f59c22e2700 1 --2- 172.21.15.154:0/656502052 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f59bc1019b0 0x7f59bc0f79b0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f59b000b820 tx=0x7f59b0009e30 comp rx=0 tx=0).stop 2024-04-02T04:51:25.987 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.983+0000 7f59c22e2700 1 -- 172.21.15.154:0/656502052 shutdown_connections 2024-04-02T04:51:25.987 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.983+0000 7f59c22e2700 1 --2- 172.21.15.154:0/656502052 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f59a4036970 0x7f59a4038e20 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:25.987 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.983+0000 7f59c22e2700 1 --2- 172.21.15.154:0/656502052 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f59bc1019b0 0x7f59bc0f79b0 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:25.987 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.983+0000 7f59c22e2700 1 -- 172.21.15.154:0/656502052 >> 172.21.15.154:0/656502052 conn(0x7f59bc0f7510 msgr2=0x7f59bc065ec0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:51:25.987 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.983+0000 7f59c22e2700 1 -- 172.21.15.154:0/656502052 shutdown_connections 2024-04-02T04:51:25.987 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:25.983+0000 7f59c22e2700 1 -- 172.21.15.154:0/656502052 wait complete. 2024-04-02T04:51:25.987 INFO:teuthology.orchestra.run.smithi154.stderr:dumped monmap epoch 1 2024-04-02T04:51:26.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:26 smithi152 ceph-mon[29073]: from='client.? 172.21.15.154:0/656502052' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:51:27.775 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T04:51:27.775 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T04:51:28.676 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T04:51:28.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:28 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:28.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:28 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:28.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:28 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:28.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:28 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:28.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:28 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T04:51:28.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:28 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:51:28.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:28 smithi152 ceph-mon[29073]: Deploying daemon mon.smithi154 on smithi154 2024-04-02T04:51:30.726 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.721+0000 7f52843a8700 1 -- 172.21.15.154:0/3499663653 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f527c0fdaa0 msgr2=0x7f527c0fde70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:30.726 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.721+0000 7f52843a8700 1 --2- 172.21.15.154:0/3499663653 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f527c0fdaa0 0x7f527c0fde70 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 crypto rx=0x7f52780048c0 tx=0x7f5278009e30 comp rx=0 tx=0).stop 2024-04-02T04:51:30.726 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.722+0000 7f52843a8700 1 -- 172.21.15.154:0/3499663653 shutdown_connections 2024-04-02T04:51:30.726 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.722+0000 7f52843a8700 1 --2- 172.21.15.154:0/3499663653 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f527c0fdaa0 0x7f527c0fde70 unknown :-1 s=CLOSED pgs=137 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:30.726 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.722+0000 7f52843a8700 1 -- 172.21.15.154:0/3499663653 >> 172.21.15.154:0/3499663653 conn(0x7f527c0f74e0 msgr2=0x7f527c0f98f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:51:30.726 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.722+0000 7f52843a8700 1 -- 172.21.15.154:0/3499663653 shutdown_connections 2024-04-02T04:51:30.727 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.722+0000 7f52843a8700 1 -- 172.21.15.154:0/3499663653 wait complete. 2024-04-02T04:51:30.727 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.722+0000 7f52843a8700 1 Processor -- start 2024-04-02T04:51:30.727 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.722+0000 7f52843a8700 1 -- start start 2024-04-02T04:51:30.727 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.722+0000 7f52843a8700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f527c0fdaa0 0x7f527c1131e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:51:30.727 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.722+0000 7f52843a8700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f527c113720 con 0x7f527c0fdaa0 2024-04-02T04:51:30.727 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.723+0000 7f5282144700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f527c0fdaa0 0x7f527c1131e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:51:30.727 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.723+0000 7f5282144700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f527c0fdaa0 0x7f527c1131e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.154:36296/0 (socket says 172.21.15.154:36296) 2024-04-02T04:51:30.727 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.723+0000 7f5282144700 1 -- 172.21.15.154:0/1262937974 learned_addr learned my addr 172.21.15.154:0/1262937974 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-04-02T04:51:30.728 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.723+0000 7f5282144700 1 -- 172.21.15.154:0/1262937974 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5278005040 con 0x7f527c0fdaa0 2024-04-02T04:51:30.729 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.723+0000 7f5282144700 1 --2- 172.21.15.154:0/1262937974 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f527c0fdaa0 0x7f527c1131e0 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f527800b820 tx=0x7f5278009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:51:30.729 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.723+0000 7f5272ffd700 1 -- 172.21.15.154:0/1262937974 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f52780093c0 con 0x7f527c0fdaa0 2024-04-02T04:51:30.729 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.723+0000 7f52843a8700 1 -- 172.21.15.154:0/1262937974 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f527c113920 con 0x7f527c0fdaa0 2024-04-02T04:51:30.729 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.723+0000 7f5272ffd700 1 -- 172.21.15.154:0/1262937974 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f5278013420 con 0x7f527c0fdaa0 2024-04-02T04:51:30.729 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.723+0000 7f5272ffd700 1 -- 172.21.15.154:0/1262937974 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f527801c5b0 con 0x7f527c0fdaa0 2024-04-02T04:51:30.729 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.723+0000 7f52843a8700 1 -- 172.21.15.154:0/1262937974 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f527c113d90 con 0x7f527c0fdaa0 2024-04-02T04:51:30.729 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.724+0000 7f5272ffd700 1 -- 172.21.15.154:0/1262937974 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 17) v1 ==== 43483+0+0 (secure 0 0 0) 0x7f5278009520 con 0x7f527c0fdaa0 2024-04-02T04:51:30.729 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.724+0000 7f52843a8700 1 -- 172.21.15.154:0/1262937974 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f527c066120 con 0x7f527c0fdaa0 2024-04-02T04:51:30.729 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.724+0000 7f5272ffd700 1 --2- 172.21.15.154:0/1262937974 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f5268036970 0x7f5268038e20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:51:30.729 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.724+0000 7f5272ffd700 1 -- 172.21.15.154:0/1262937974 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1366+0+0 (secure 0 0 0) 0x7f5278049b10 con 0x7f527c0fdaa0 2024-04-02T04:51:30.729 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.725+0000 7f5281943700 1 --2- 172.21.15.154:0/1262937974 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f5268036970 0x7f5268038e20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:51:30.729 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.725+0000 7f5281943700 1 --2- 172.21.15.154:0/1262937974 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f5268036970 0x7f5268038e20 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f526c009990 tx=0x7f526c006e40 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:51:30.732 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:30.729+0000 7f5272ffd700 1 -- 172.21.15.154:0/1262937974 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f5278013a90 con 0x7f527c0fdaa0 2024-04-02T04:51:31.007 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:31.004+0000 7f52843a8700 1 -- 172.21.15.154:0/1262937974 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f527c05f5f0 con 0x7f527c0fdaa0 2024-04-02T04:51:31.008 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:31.005+0000 7f5272ffd700 1 -- 172.21.15.154:0/1262937974 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+743 (secure 0 0 0) 0x7f5278011370 con 0x7f527c0fdaa0 2024-04-02T04:51:31.008 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:51:31.008 INFO:teuthology.orchestra.run.smithi154.stdout:{"epoch":1,"fsid":"2e0055d0-f0ac-11ee-b647-cb9ed24678a4","modified":"2024-04-02T04:48:56.938040Z","created":"2024-04-02T04:48:56.938040Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi152","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:3300","nonce":0},{"type":"v1","addr":"172.21.15.152:6789","nonce":0}]},"addr":"172.21.15.152:6789/0","public_addr":"172.21.15.152:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-02T04:51:31.010 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:31.007+0000 7f52843a8700 1 -- 172.21.15.154:0/1262937974 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f5268036970 msgr2=0x7f5268038e20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:31.010 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:31.007+0000 7f52843a8700 1 --2- 172.21.15.154:0/1262937974 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f5268036970 0x7f5268038e20 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 crypto rx=0x7f526c009990 tx=0x7f526c006e40 comp rx=0 tx=0).stop 2024-04-02T04:51:31.010 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:31.007+0000 7f52843a8700 1 -- 172.21.15.154:0/1262937974 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f527c0fdaa0 msgr2=0x7f527c1131e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:31.011 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:31.007+0000 7f52843a8700 1 --2- 172.21.15.154:0/1262937974 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f527c0fdaa0 0x7f527c1131e0 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 crypto rx=0x7f527800b820 tx=0x7f5278009e30 comp rx=0 tx=0).stop 2024-04-02T04:51:31.011 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:31.007+0000 7f52843a8700 1 -- 172.21.15.154:0/1262937974 shutdown_connections 2024-04-02T04:51:31.011 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:31.007+0000 7f52843a8700 1 --2- 172.21.15.154:0/1262937974 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f5268036970 0x7f5268038e20 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:31.011 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:31.007+0000 7f52843a8700 1 --2- 172.21.15.154:0/1262937974 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f527c0fdaa0 0x7f527c1131e0 unknown :-1 s=CLOSED pgs=138 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:31.011 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:31.007+0000 7f52843a8700 1 -- 172.21.15.154:0/1262937974 >> 172.21.15.154:0/1262937974 conn(0x7f527c0f74e0 msgr2=0x7f527c1038c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:51:31.011 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:31.007+0000 7f52843a8700 1 -- 172.21.15.154:0/1262937974 shutdown_connections 2024-04-02T04:51:31.011 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:31.007+0000 7f52843a8700 1 -- 172.21.15.154:0/1262937974 wait complete. 2024-04-02T04:51:31.011 INFO:teuthology.orchestra.run.smithi154.stderr:dumped monmap epoch 1 2024-04-02T04:51:31.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:31 smithi152 ceph-mon[29073]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:31.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:31 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.117 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 podman[35706]: 2024-04-02 04:51:31.985278818 +0000 UTC m=+0.019221741 image pull 4948510fac0f2ac7df4b4e361573714a48c110d48535a4b8d5aeb81e8023bedd quay.ceph.io/ceph-ci/ceph@sha256:afcab66d4788c00ea37075a6986aaec87a4f099e0d803aec3d45584d18bd8a6e 2024-04-02T04:51:32.117 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 podman[35706]: 2024-04-02 04:51:32.113763674 +0000 UTC m=+0.147706596 container create 975aefa2215f8196052cd01ff7d6b6b136643ec0d5b9c5be9aa01578fd888981 (image=quay.ceph.io/ceph-ci/ceph@sha256:afcab66d4788c00ea37075a6986aaec87a4f099e0d803aec3d45584d18bd8a6e, name=ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-mon-smithi154, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-c924f99, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2) 2024-04-02T04:51:32.376 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 podman[35706]: 2024-04-02 04:51:32.349472629 +0000 UTC m=+0.383415566 container init 975aefa2215f8196052cd01ff7d6b6b136643ec0d5b9c5be9aa01578fd888981 (image=quay.ceph.io/ceph-ci/ceph@sha256:afcab66d4788c00ea37075a6986aaec87a4f099e0d803aec3d45584d18bd8a6e, name=ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-mon-smithi154, org.label-schema.vendor=CentOS, RELEASE=quincy-c924f99, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-04-02T04:51:32.376 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 podman[35706]: 2024-04-02 04:51:32.353670129 +0000 UTC m=+0.387613053 container start 975aefa2215f8196052cd01ff7d6b6b136643ec0d5b9c5be9aa01578fd888981 (image=quay.ceph.io/ceph-ci/ceph@sha256:afcab66d4788c00ea37075a6986aaec87a4f099e0d803aec3d45584d18bd8a6e, name=ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-mon-smithi154, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=quincy-c924f99, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, GIT_COMMIT=12717c0777377369ea674892da98b0d85250f5b0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-02T04:51:32.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:32 smithi152 ceph-mon[29073]: from='client.? 172.21.15.154:0/1262937974' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:51:32.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: set uid:gid to 167:167 (ceph:ceph) 2024-04-02T04:51:32.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: ceph version 17.2.7-803-gc924f991 (c924f9914f2809997b363f77c5cbb14929881220) quincy (stable), process ceph-mon, pid 2 2024-04-02T04:51:32.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: pidfile_write: ignore empty --pid-file 2024-04-02T04:51:32.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: load: jerasure load: lrc 2024-04-02T04:51:32.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: RocksDB version: 6.15.5 2024-04-02T04:51:32.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-02T04:51:32.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Compile date Apr 1 2024 2024-04-02T04:51:32.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: DB SUMMARY 2024-04-02T04:51:32.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: DB Session ID: 7UBGJDFGE466GU4B7HKL 2024-04-02T04:51:32.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: CURRENT file: CURRENT 2024-04-02T04:51:32.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: IDENTITY file: IDENTITY 2024-04-02T04:51:32.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-02T04:51:32.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi154/store.db dir, Total Num: 0, files: 2024-04-02T04:51:32.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi154/store.db: 000004.log size: 511 ; 2024-04-02T04:51:32.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.error_if_exists: 0 2024-04-02T04:51:32.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.create_if_missing: 0 2024-04-02T04:51:32.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.paranoid_checks: 1 2024-04-02T04:51:32.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-02T04:51:32.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.env: 0x557866032a80 2024-04-02T04:51:32.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.fs: Posix File System 2024-04-02T04:51:32.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.info_log: 0x5578680b1c00 2024-04-02T04:51:32.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_file_opening_threads: 16 2024-04-02T04:51:32.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.statistics: (nil) 2024-04-02T04:51:32.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.use_fsync: 0 2024-04-02T04:51:32.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_log_file_size: 0 2024-04-02T04:51:32.736 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-02T04:51:32.736 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-02T04:51:32.736 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.keep_log_file_num: 1000 2024-04-02T04:51:32.736 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.recycle_log_file_num: 0 2024-04-02T04:51:32.736 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.allow_fallocate: 1 2024-04-02T04:51:32.736 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.allow_mmap_reads: 0 2024-04-02T04:51:32.736 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.allow_mmap_writes: 0 2024-04-02T04:51:32.736 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.use_direct_reads: 0 2024-04-02T04:51:32.736 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-02T04:51:32.736 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.create_missing_column_families: 0 2024-04-02T04:51:32.736 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.db_log_dir: 2024-04-02T04:51:32.736 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi154/store.db 2024-04-02T04:51:32.737 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-02T04:51:32.737 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-02T04:51:32.739 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-02T04:51:32.739 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-02T04:51:32.739 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-02T04:51:32.739 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-02T04:51:32.739 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.advise_random_on_open: 1 2024-04-02T04:51:32.739 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.db_write_buffer_size: 0 2024-04-02T04:51:32.739 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.write_buffer_manager: 0x557868e26990 2024-04-02T04:51:32.739 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-02T04:51:32.740 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-02T04:51:32.740 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-02T04:51:32.740 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-02T04:51:32.740 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.rate_limiter: (nil) 2024-04-02T04:51:32.740 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-02T04:51:32.740 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.wal_recovery_mode: 2 2024-04-02T04:51:32.740 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.enable_thread_tracking: 0 2024-04-02T04:51:32.740 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.enable_pipelined_write: 0 2024-04-02T04:51:32.740 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.unordered_write: 0 2024-04-02T04:51:32.740 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-02T04:51:32.740 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-02T04:51:32.740 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-02T04:51:32.740 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-02T04:51:32.740 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.row_cache: None 2024-04-02T04:51:32.740 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.wal_filter: None 2024-04-02T04:51:32.741 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-02T04:51:32.741 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.allow_ingest_behind: 0 2024-04-02T04:51:32.741 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.preserve_deletes: 0 2024-04-02T04:51:32.741 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.two_write_queues: 0 2024-04-02T04:51:32.741 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.manual_wal_flush: 0 2024-04-02T04:51:32.741 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.atomic_flush: 0 2024-04-02T04:51:32.741 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-02T04:51:32.741 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-02T04:51:32.741 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-02T04:51:32.741 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.log_readahead_size: 0 2024-04-02T04:51:32.741 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-02T04:51:32.741 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.best_efforts_recovery: 0 2024-04-02T04:51:32.741 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-02T04:51:32.741 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-02T04:51:32.742 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.allow_data_in_errors: 0 2024-04-02T04:51:32.742 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.db_host_id: __hostname__ 2024-04-02T04:51:32.742 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_background_jobs: 2 2024-04-02T04:51:32.742 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_background_compactions: -1 2024-04-02T04:51:32.742 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_subcompactions: 1 2024-04-02T04:51:32.742 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-02T04:51:32.742 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-02T04:51:32.742 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-02T04:51:32.742 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_total_wal_size: 0 2024-04-02T04:51:32.742 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-02T04:51:32.742 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-02T04:51:32.743 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-02T04:51:32.743 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-02T04:51:32.743 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_open_files: -1 2024-04-02T04:51:32.743 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.bytes_per_sync: 0 2024-04-02T04:51:32.743 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-02T04:51:32.743 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-02T04:51:32.743 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.compaction_readahead_size: 0 2024-04-02T04:51:32.743 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_background_flushes: -1 2024-04-02T04:51:32.743 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Compression algorithms supported: 2024-04-02T04:51:32.743 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-02T04:51:32.743 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: kZSTD supported: 0 2024-04-02T04:51:32.743 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: kXpressCompression supported: 0 2024-04-02T04:51:32.744 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: kLZ4HCCompression supported: 1 2024-04-02T04:51:32.744 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: kZlibCompression supported: 1 2024-04-02T04:51:32.744 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: kSnappyCompression supported: 1 2024-04-02T04:51:32.744 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: kLZ4Compression supported: 1 2024-04-02T04:51:32.744 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: kBZip2Compression supported: 0 2024-04-02T04:51:32.744 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-02T04:51:32.744 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi154/store.db/MANIFEST-000003 2024-04-02T04:51:32.744 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-04-02T04:51:32.744 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-02T04:51:32.744 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.merge_operator: 2024-04-02T04:51:32.744 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.compaction_filter: None 2024-04-02T04:51:32.744 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.compaction_filter_factory: None 2024-04-02T04:51:32.744 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.sst_partitioner_factory: None 2024-04-02T04:51:32.744 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-02T04:51:32.744 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-02T04:51:32.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x557868106f58) 2024-04-02T04:51:32.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: cache_index_and_filter_blocks: 1 2024-04-02T04:51:32.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-02T04:51:32.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-02T04:51:32.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: pin_top_level_index_and_filter: 1 2024-04-02T04:51:32.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: index_type: 0 2024-04-02T04:51:32.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: data_block_index_type: 0 2024-04-02T04:51:32.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: index_shortening: 1 2024-04-02T04:51:32.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-02T04:51:32.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: hash_index_allow_collision: 1 2024-04-02T04:51:32.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: checksum: 1 2024-04-02T04:51:32.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: no_block_cache: 0 2024-04-02T04:51:32.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: block_cache: 0x557868139090 2024-04-02T04:51:32.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: block_cache_name: BinnedLRUCache 2024-04-02T04:51:32.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: block_cache_options: 2024-04-02T04:51:32.745 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: capacity : 536870912 2024-04-02T04:51:32.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: num_shard_bits : 4 2024-04-02T04:51:32.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: strict_capacity_limit : 0 2024-04-02T04:51:32.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: high_pri_pool_ratio: 0.000 2024-04-02T04:51:32.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: block_cache_compressed: (nil) 2024-04-02T04:51:32.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: persistent_cache: (nil) 2024-04-02T04:51:32.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: block_size: 4096 2024-04-02T04:51:32.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: block_size_deviation: 10 2024-04-02T04:51:32.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: block_restart_interval: 16 2024-04-02T04:51:32.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: index_block_restart_interval: 1 2024-04-02T04:51:32.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: metadata_block_size: 4096 2024-04-02T04:51:32.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: partition_filters: 0 2024-04-02T04:51:32.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: use_delta_encoding: 1 2024-04-02T04:51:32.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-02T04:51:32.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: whole_key_filtering: 1 2024-04-02T04:51:32.746 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: verify_compression: 0 2024-04-02T04:51:32.747 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: read_amp_bytes_per_bit: 0 2024-04-02T04:51:32.747 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: format_version: 4 2024-04-02T04:51:32.747 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: enable_index_compression: 1 2024-04-02T04:51:32.747 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: block_align: 0 2024-04-02T04:51:32.747 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.write_buffer_size: 33554432 2024-04-02T04:51:32.747 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_write_buffer_number: 2 2024-04-02T04:51:32.747 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.compression: NoCompression 2024-04-02T04:51:32.747 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.bottommost_compression: Disabled 2024-04-02T04:51:32.747 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.prefix_extractor: nullptr 2024-04-02T04:51:32.747 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-02T04:51:32.747 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.num_levels: 7 2024-04-02T04:51:32.747 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-02T04:51:32.747 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-02T04:51:32.747 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-02T04:51:32.748 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-02T04:51:32.748 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-02T04:51:32.748 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-02T04:51:32.748 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-02T04:51:32.748 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-02T04:51:32.748 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-02T04:51:32.748 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-02T04:51:32.748 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-02T04:51:32.748 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.compression_opts.level: 32767 2024-04-02T04:51:32.748 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.compression_opts.strategy: 0 2024-04-02T04:51:32.748 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-02T04:51:32.748 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-02T04:51:32.748 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-02T04:51:32.748 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.compression_opts.enabled: false 2024-04-02T04:51:32.749 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-02T04:51:32.749 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-02T04:51:32.749 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-02T04:51:32.749 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.target_file_size_base: 67108864 2024-04-02T04:51:32.749 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-02T04:51:32.749 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-02T04:51:32.749 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-02T04:51:32.749 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-02T04:51:32.749 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-02T04:51:32.749 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-02T04:51:32.749 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-02T04:51:32.749 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-02T04:51:32.749 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-02T04:51:32.749 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-02T04:51:32.749 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-02T04:51:32.750 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-02T04:51:32.750 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-02T04:51:32.750 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.arena_block_size: 4194304 2024-04-02T04:51:32.750 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-02T04:51:32.750 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-02T04:51:32.750 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-02T04:51:32.750 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.disable_auto_compactions: 0 2024-04-02T04:51:32.750 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-02T04:51:32.750 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-02T04:51:32.750 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-02T04:51:32.750 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-02T04:51:32.750 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-02T04:51:32.750 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-02T04:51:32.750 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-02T04:51:32.750 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-02T04:51:32.750 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-02T04:51:32.750 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-02T04:51:32.751 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.table_properties_collectors: 2024-04-02T04:51:32.751 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.inplace_update_support: 0 2024-04-02T04:51:32.751 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-02T04:51:32.751 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-02T04:51:32.751 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-02T04:51:32.751 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-02T04:51:32.751 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.bloom_locality: 0 2024-04-02T04:51:32.751 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.max_successive_merges: 0 2024-04-02T04:51:32.751 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-02T04:51:32.751 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.paranoid_file_checks: 0 2024-04-02T04:51:32.751 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.force_consistency_checks: 1 2024-04-02T04:51:32.751 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.report_bg_io_stats: 0 2024-04-02T04:51:32.751 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.ttl: 2592000 2024-04-02T04:51:32.751 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-02T04:51:32.751 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.enable_blob_files: false 2024-04-02T04:51:32.751 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.min_blob_size: 0 2024-04-02T04:51:32.752 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.blob_file_size: 268435456 2024-04-02T04:51:32.752 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-02T04:51:32.752 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-02T04:51:32.752 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-02T04:51:32.752 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi154/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-02T04:51:32.752 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-04-02T04:51:32.752 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-04-02T04:51:32.752 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712033492380056, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-02T04:51:32.752 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-04-02T04:51:32.752 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-02T04:51:32.752 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712033492380375, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1712033492, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "6714c745-4df3-4ed5-adb1-1e07cf664a33", "db_session_id": "7UBGJDFGE466GU4B7HKL"}} 2024-04-02T04:51:32.752 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-04-02T04:51:32.752 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: EVENT_LOG_v1 {"time_micros": 1712033492380740, "job": 1, "event": "recovery_finished"} 2024-04-02T04:51:32.752 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-smithi154/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-02T04:51:32.752 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5578680cd180 2024-04-02T04:51:32.753 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: DB pointer 0x5578681a2000 2024-04-02T04:51:32.753 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-04-02T04:51:32.753 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-02T04:51:32.753 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ** DB Stats ** 2024-04-02T04:51:32.753 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T04:51:32.753 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-02T04:51:32.753 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-02T04:51:32.753 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-02T04:51:32.753 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-02T04:51:32.753 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-02T04:51:32.753 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-02T04:51:32.753 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: 2024-04-02T04:51:32.753 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ** Compaction Stats [default] ** 2024-04-02T04:51:32.753 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T04:51:32.753 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T04:51:32.754 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-04-02T04:51:32.754 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-04-02T04:51:32.754 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-04-02T04:51:32.754 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: 2024-04-02T04:51:32.754 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ** Compaction Stats [default] ** 2024-04-02T04:51:32.754 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T04:51:32.754 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T04:51:32.754 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-04-02T04:51:32.754 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T04:51:32.754 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-02T04:51:32.754 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-02T04:51:32.754 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-02T04:51:32.754 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-02T04:51:32.754 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-02T04:51:32.755 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Cumulative compaction: 0.00 GB write, 0.53 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T04:51:32.755 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Interval compaction: 0.00 GB write, 0.53 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T04:51:32.755 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-02T04:51:32.755 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: 2024-04-02T04:51:32.755 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-02T04:51:32.755 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: 2024-04-02T04:51:32.755 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ** Compaction Stats [default] ** 2024-04-02T04:51:32.755 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T04:51:32.755 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T04:51:32.755 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-04-02T04:51:32.755 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-04-02T04:51:32.755 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-02T04:51:32.755 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: 2024-04-02T04:51:32.755 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ** Compaction Stats [default] ** 2024-04-02T04:51:32.755 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-02T04:51:32.755 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-02T04:51:32.756 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.8 0.00 0.00 1 0.000 0 0 2024-04-02T04:51:32.756 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-02T04:51:32.756 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-02T04:51:32.756 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-02T04:51:32.756 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-02T04:51:32.756 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-02T04:51:32.756 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-02T04:51:32.756 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Cumulative compaction: 0.00 GB write, 0.52 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T04:51:32.756 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-02T04:51:32.756 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-02T04:51:32.756 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: 2024-04-02T04:51:32.756 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-02T04:51:32.756 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: mon.smithi154 does not exist in monmap, will attempt to join an existing cluster 2024-04-02T04:51:32.756 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: using public_addr v2:172.21.15.154:0/0 -> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] 2024-04-02T04:51:32.756 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: starting mon.smithi154 rank -1 at public addrs [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] at bind addrs [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi154 fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 2024-04-02T04:51:32.756 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: mon.smithi154@-1(???) e0 preinit fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 2024-04-02T04:51:32.756 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: mon.smithi154@-1(synchronizing).mds e1 new map 2024-04-02T04:51:32.757 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: mon.smithi154@-1(synchronizing).mds e1 print_map 2024-04-02T04:51:32.757 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: e1 2024-04-02T04:51:32.757 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-02T04:51:32.757 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-02T04:51:32.757 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: legacy client fscid: -1 2024-04-02T04:51:32.757 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: 2024-04-02T04:51:32.757 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout: No filesystems configured 2024-04-02T04:51:32.757 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: mon.smithi154@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-02T04:51:32.757 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: mon.smithi154@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-02T04:51:32.757 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: mon.smithi154@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-02T04:51:32.757 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: mon.smithi154@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-02T04:51:32.757 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: mon.smithi154@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-02T04:51:32.757 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: mon.smithi154@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-02T04:51:32.757 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: mon.smithi154@-1(synchronizing).osd e5 e5: 0 total, 0 up, 0 in 2024-04-02T04:51:32.757 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: mon.smithi154@-1(synchronizing).osd e5 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-02T04:51:32.758 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: mon.smithi154@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T04:51:32.758 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: mon.smithi154@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T04:51:32.758 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: mon.smithi154@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2024-04-02T04:51:32.758 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi154", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:51:32.758 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: Deploying cephadm binary to smithi154 2024-04-02T04:51:32.758 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:32.758 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.758 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.758 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: Added host smithi154 2024-04-02T04:51:32.758 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.758 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.758 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.759 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.759 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: Deploying daemon alertmanager.smithi152 on smithi152 2024-04-02T04:51:32.759 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:32.759 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:32.759 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T04:51:32.759 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.759 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:32.759 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:32.759 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/2209820146' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-02T04:51:32.759 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/2209820146' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-02T04:51:32.759 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: osdmap e4: 0 total, 0 up, 0 in 2024-04-02T04:51:32.759 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:32.759 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:32.759 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.759 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.759 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.760 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.760 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.760 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.760 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T04:51:32.760 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-02T04:51:32.760 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.760 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: Deploying daemon grafana.smithi152 on smithi152 2024-04-02T04:51:32.760 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='client.14192 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi152:172.21.15.152=smithi152;smithi154:172.21.15.154=smithi154", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:51:32.760 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: Saving service mon spec with placement smithi152:172.21.15.152=smithi152;smithi154:172.21.15.154=smithi154;count:2 2024-04-02T04:51:32.760 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.760 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:32.760 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.760 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:32.760 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='client.? 172.21.15.154:0/58708382' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:51:32.760 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:32.760 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='client.? 172.21.15.154:0/142823751' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:51:32.761 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:32.761 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:32.761 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='client.? 172.21.15.154:0/3950751420' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:51:32.761 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:32.761 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.761 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.761 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.761 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.761 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.761 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.761 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: Deploying daemon prometheus.smithi152 on smithi152 2024-04-02T04:51:32.761 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:32.761 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='client.? 172.21.15.154:0/2747056202' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:51:32.761 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:32.761 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:32.762 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.762 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='client.? 172.21.15.154:0/2162923981' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:51:32.762 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:32.762 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:32.762 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='client.? 172.21.15.154:0/2183053541' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:51:32.762 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:32.762 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='client.? 172.21.15.154:0/2750843789' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:51:32.762 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:32.762 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:32.762 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.762 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.762 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.762 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-02T04:51:32.762 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='client.? 172.21.15.154:0/233656300' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:51:32.762 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14160 172.21.15.152:0/3235146215' entity='mgr.smithi152.jivdcl' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-02T04:51:32.762 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: mgrmap e14: smithi152.jivdcl(active, since 61s) 2024-04-02T04:51:32.762 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='client.? 172.21.15.154:0/1755549450' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:51:32.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: Active manager daemon smithi152.jivdcl restarted 2024-04-02T04:51:32.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: Activating manager daemon smithi152.jivdcl 2024-04-02T04:51:32.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: osdmap e5: 0 total, 0 up, 0 in 2024-04-02T04:51:32.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: mgrmap e15: smithi152.jivdcl(active, starting, since 0.0033924s) 2024-04-02T04:51:32.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi152"}]: dispatch 2024-04-02T04:51:32.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mgr metadata", "who": "smithi152.jivdcl", "id": "smithi152.jivdcl"}]: dispatch 2024-04-02T04:51:32.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-02T04:51:32.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-02T04:51:32.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-02T04:51:32.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: Manager daemon smithi152.jivdcl is now available 2024-04-02T04:51:32.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:51:32.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:51:32.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:51:32.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi152.jivdcl/mirror_snapshot_schedule"}]: dispatch 2024-04-02T04:51:32.763 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi152.jivdcl/trash_purge_schedule"}]: dispatch 2024-04-02T04:51:32.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='client.? 172.21.15.154:0/2454352752' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:51:32.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: mgrmap e16: smithi152.jivdcl(active, since 1.00831s) 2024-04-02T04:51:32.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: [02/Apr/2024:04:51:10] ENGINE Bus STARTING 2024-04-02T04:51:32.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: [02/Apr/2024:04:51:11] ENGINE Serving on https://172.21.15.152:7150 2024-04-02T04:51:32.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: [02/Apr/2024:04:51:11] ENGINE Bus STARTED 2024-04-02T04:51:32.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: mgrmap e17: smithi152.jivdcl(active, since 2s) 2024-04-02T04:51:32.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd/host:smithi152", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:51:32.764 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd/host:smithi154", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:51:32.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:51:32.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:51:32.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-02T04:51:32.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: Updating smithi154:/etc/ceph/ceph.conf 2024-04-02T04:51:32.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: Updating smithi152:/var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T04:51:32.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: Updating smithi154:/var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T04:51:32.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: Updating smithi152:/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:51:32.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: Updating smithi154:/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:51:32.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: Updating smithi152:/var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T04:51:32.765 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: Updating smithi154:/var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/config/ceph.client.admin.keyring 2024-04-02T04:51:32.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi154", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-02T04:51:32.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi154", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-02T04:51:32.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:51:32.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: Deploying daemon crash.smithi154 on smithi154 2024-04-02T04:51:32.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='client.? 172.21.15.154:0/229313689' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:51:32.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.766 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: Deploying daemon node-exporter.smithi154 on smithi154 2024-04-02T04:51:32.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='client.? 172.21.15.154:0/1791533700' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:51:32.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi154.pplnxz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T04:51:32.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi154.pplnxz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-02T04:51:32.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T04:51:32.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:51:32.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: Deploying daemon mgr.smithi154.pplnxz on smithi154 2024-04-02T04:51:32.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.767 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:51:32.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='client.? 172.21.15.154:0/656502052' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:51:32.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T04:51:32.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:51:32.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: Deploying daemon mon.smithi154 on smithi154 2024-04-02T04:51:32.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:32.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:32.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: from='client.? 172.21.15.154:0/1262937974' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:51:32.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 ceph-mon[35729]: mon.smithi154@-1(synchronizing).paxosservice(auth 1..6) refresh upgraded, format 0 -> 3 2024-04-02T04:51:32.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 bash[35706]: 975aefa2215f8196052cd01ff7d6b6b136643ec0d5b9c5be9aa01578fd888981 2024-04-02T04:51:32.768 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:32 smithi154 systemd[1]: Started Ceph mon.smithi154 for 2e0055d0-f0ac-11ee-b647-cb9ed24678a4. 2024-04-02T04:51:32.874 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-02T04:51:32.874 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph mon dump -f json 2024-04-02T04:51:33.135 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi154/config 2024-04-02T04:51:37.636 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.630+0000 7fd700682700 1 -- 172.21.15.154:0/2063405641 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd6e4003580 msgr2=0x7fd6e4005a00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:37.636 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.630+0000 7fd700682700 1 --2- 172.21.15.154:0/2063405641 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd6e4003580 0x7fd6e4005a00 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 crypto rx=0x7fd6f8065f40 tx=0x7fd6f400b040 comp rx=0 tx=0).stop 2024-04-02T04:51:37.636 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.631+0000 7fd700682700 1 -- 172.21.15.154:0/2063405641 shutdown_connections 2024-04-02T04:51:37.637 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.631+0000 7fd700682700 1 --2- 172.21.15.154:0/2063405641 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd6e4003580 0x7fd6e4005a00 unknown :-1 s=CLOSED pgs=142 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:37.637 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.631+0000 7fd700682700 1 -- 172.21.15.154:0/2063405641 >> 172.21.15.154:0/2063405641 conn(0x7fd6f80f75f0 msgr2=0x7fd6f80f9a00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:51:37.637 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.631+0000 7fd700682700 1 -- 172.21.15.154:0/2063405641 shutdown_connections 2024-04-02T04:51:37.637 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.631+0000 7fd700682700 1 -- 172.21.15.154:0/2063405641 wait complete. 2024-04-02T04:51:37.637 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.631+0000 7fd700682700 1 Processor -- start 2024-04-02T04:51:37.637 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.631+0000 7fd700682700 1 -- start start 2024-04-02T04:51:37.637 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.632+0000 7fd700682700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd6e4003580 0x7fd6f8113330 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:51:37.637 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.632+0000 7fd700682700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd6f8113870 0x7fd6f810d290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:51:37.637 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.632+0000 7fd700682700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd6f8113e80 con 0x7fd6e4003580 2024-04-02T04:51:37.637 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.632+0000 7fd700682700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd6f810d7d0 con 0x7fd6f8113870 2024-04-02T04:51:37.638 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.632+0000 7fd6fdc1d700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd6f8113870 0x7fd6f810d290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:51:37.638 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.632+0000 7fd6fe41e700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd6e4003580 0x7fd6f8113330 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:51:37.638 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.632+0000 7fd6fdc1d700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd6f8113870 0x7fd6f810d290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.154:57400/0 (socket says 172.21.15.154:57400) 2024-04-02T04:51:37.638 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.632+0000 7fd6fdc1d700 1 -- 172.21.15.154:0/3550267504 learned_addr learned my addr 172.21.15.154:0/3550267504 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-04-02T04:51:37.638 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.632+0000 7fd6fe41e700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd6e4003580 0x7fd6f8113330 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.154:45342/0 (socket says 172.21.15.154:45342) 2024-04-02T04:51:37.638 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.632+0000 7fd6fdc1d700 1 -- 172.21.15.154:0/3550267504 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd6f8113870 msgr2=0x7fd6f810d290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_bulk peer close file descriptor 13 2024-04-02T04:51:37.638 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.632+0000 7fd6fdc1d700 1 -- 172.21.15.154:0/3550267504 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd6f8113870 msgr2=0x7fd6f810d290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).read_until read failed 2024-04-02T04:51:37.638 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.632+0000 7fd6fdc1d700 1 --2- 172.21.15.154:0/3550267504 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd6f8113870 0x7fd6f810d290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_read_frame_preamble_main read frame preamble failed r=-1 ((1) Operation not permitted) 2024-04-02T04:51:37.638 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.632+0000 7fd6fdc1d700 1 --2- 172.21.15.154:0/3550267504 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd6f8113870 0x7fd6f810d290 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0)._fault waiting 0.200000 2024-04-02T04:51:37.638 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.632+0000 7fd6fe41e700 1 -- 172.21.15.154:0/3550267504 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd6f8113870 msgr2=0x7fd6f810d290 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2024-04-02T04:51:37.638 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.632+0000 7fd6fe41e700 1 --2- 172.21.15.154:0/3550267504 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd6f8113870 0x7fd6f810d290 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:37.639 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.632+0000 7fd6fe41e700 1 -- 172.21.15.154:0/3550267504 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd6f4005040 con 0x7fd6e4003580 2024-04-02T04:51:37.639 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.633+0000 7fd6fe41e700 1 --2- 172.21.15.154:0/3550267504 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd6e4003580 0x7fd6f8113330 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7fd6f4007c90 tx=0x7fd6f400b040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:51:37.639 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.633+0000 7fd6eeffd700 1 -- 172.21.15.154:0/3550267504 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fd6f401b070 con 0x7fd6e4003580 2024-04-02T04:51:37.639 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.633+0000 7fd6eeffd700 1 -- 172.21.15.154:0/3550267504 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fd6f4020910 con 0x7fd6e4003580 2024-04-02T04:51:37.639 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.633+0000 7fd700682700 1 -- 172.21.15.154:0/3550267504 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd6f810da50 con 0x7fd6e4003580 2024-04-02T04:51:37.639 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.633+0000 7fd6eeffd700 1 -- 172.21.15.154:0/3550267504 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fd6f401f700 con 0x7fd6e4003580 2024-04-02T04:51:37.639 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.633+0000 7fd700682700 1 -- 172.21.15.154:0/3550267504 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd6f810df90 con 0x7fd6e4003580 2024-04-02T04:51:37.640 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.634+0000 7fd700682700 1 -- 172.21.15.154:0/3550267504 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd6f8066740 con 0x7fd6e4003580 2024-04-02T04:51:37.640 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.634+0000 7fd6eeffd700 1 -- 172.21.15.154:0/3550267504 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 18) v1 ==== 86567+0+0 (secure 0 0 0) 0x7fd6f4020440 con 0x7fd6e4003580 2024-04-02T04:51:37.640 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.635+0000 7fd6eeffd700 1 --2- 172.21.15.154:0/3550267504 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd6dc069360 0x7fd6dc06b810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:51:37.640 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.635+0000 7fd6eeffd700 1 -- 172.21.15.154:0/3550267504 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1366+0+0 (secure 0 0 0) 0x7fd6f4087db0 con 0x7fd6e4003580 2024-04-02T04:51:37.640 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.635+0000 7fd6fdc1d700 1 --2- 172.21.15.154:0/3550267504 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd6dc069360 0x7fd6dc06b810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:51:37.640 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.635+0000 7fd6fdc1d700 1 --2- 172.21.15.154:0/3550267504 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd6dc069360 0x7fd6dc06b810 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fd6e8006fd0 tx=0x7fd6e8008040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:51:37.642 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.639+0000 7fd6eeffd700 1 -- 172.21.15.154:0/3550267504 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fd6f408c070 con 0x7fd6e4003580 2024-04-02T04:51:37.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:37 smithi154 ceph-mon[35729]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:37.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:37 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi152"}]: dispatch 2024-04-02T04:51:37.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:37 smithi154 ceph-mon[35729]: mon.smithi152 calling monitor election 2024-04-02T04:51:37.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:37 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-04-02T04:51:37.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:37 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-04-02T04:51:37.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:37 smithi154 ceph-mon[35729]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:37.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:37 smithi154 ceph-mon[35729]: from='mgr.? 172.21.15.154:0/883741497' entity='mgr.smithi154.pplnxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi154.pplnxz/crt"}]: dispatch 2024-04-02T04:51:37.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:37 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-04-02T04:51:37.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:37 smithi154 ceph-mon[35729]: mon.smithi154 calling monitor election 2024-04-02T04:51:37.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:37 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-04-02T04:51:37.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:37 smithi154 ceph-mon[35729]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:37.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:37 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-04-02T04:51:37.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:37 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-04-02T04:51:37.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:37 smithi154 ceph-mon[35729]: mon.smithi152 is new leader, mons smithi152,smithi154 in quorum (ranks 0,1) 2024-04-02T04:51:37.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:37 smithi154 ceph-mon[35729]: monmap e2: 2 mons at {smithi152=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0],smithi154=[v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-02T04:51:37.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:37 smithi154 ceph-mon[35729]: fsmap 2024-04-02T04:51:37.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:37 smithi154 ceph-mon[35729]: osdmap e5: 0 total, 0 up, 0 in 2024-04-02T04:51:37.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:37 smithi154 ceph-mon[35729]: mgrmap e17: smithi152.jivdcl(active, since 27s) 2024-04-02T04:51:37.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:37 smithi154 ceph-mon[35729]: overall HEALTH_OK 2024-04-02T04:51:37.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:37 smithi154 ceph-mon[35729]: Standby manager daemon smithi154.pplnxz started 2024-04-02T04:51:37.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:37 smithi154 ceph-mon[35729]: from='mgr.? 172.21.15.154:0/883741497' entity='mgr.smithi154.pplnxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T04:51:37.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:37 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:37.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:37 smithi154 ceph-mon[35729]: from='mgr.? 172.21.15.154:0/883741497' entity='mgr.smithi154.pplnxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi154.pplnxz/key"}]: dispatch 2024-04-02T04:51:37.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:37 smithi154 ceph-mon[35729]: from='mgr.? 172.21.15.154:0/883741497' entity='mgr.smithi154.pplnxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T04:51:37.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:37 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:37.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:37 smithi152 ceph-mon[29073]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:37.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:37 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi152"}]: dispatch 2024-04-02T04:51:37.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:37 smithi152 ceph-mon[29073]: mon.smithi152 calling monitor election 2024-04-02T04:51:37.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:37 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-04-02T04:51:37.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:37 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-04-02T04:51:37.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:37 smithi152 ceph-mon[29073]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:37.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:37 smithi152 ceph-mon[29073]: from='mgr.? 172.21.15.154:0/883741497' entity='mgr.smithi154.pplnxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi154.pplnxz/crt"}]: dispatch 2024-04-02T04:51:37.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:37 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-04-02T04:51:37.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:37 smithi152 ceph-mon[29073]: mon.smithi154 calling monitor election 2024-04-02T04:51:37.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:37 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-04-02T04:51:37.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:37 smithi152 ceph-mon[29073]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:37.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:37 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-04-02T04:51:37.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:37 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-04-02T04:51:37.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:37 smithi152 ceph-mon[29073]: mon.smithi152 is new leader, mons smithi152,smithi154 in quorum (ranks 0,1) 2024-04-02T04:51:37.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:37 smithi152 ceph-mon[29073]: monmap e2: 2 mons at {smithi152=[v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0],smithi154=[v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-02T04:51:37.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:37 smithi152 ceph-mon[29073]: fsmap 2024-04-02T04:51:37.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:37 smithi152 ceph-mon[29073]: osdmap e5: 0 total, 0 up, 0 in 2024-04-02T04:51:37.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:37 smithi152 ceph-mon[29073]: mgrmap e17: smithi152.jivdcl(active, since 27s) 2024-04-02T04:51:37.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:37 smithi152 ceph-mon[29073]: overall HEALTH_OK 2024-04-02T04:51:37.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:37 smithi152 ceph-mon[29073]: Standby manager daemon smithi154.pplnxz started 2024-04-02T04:51:37.759 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:37 smithi152 ceph-mon[29073]: from='mgr.? 172.21.15.154:0/883741497' entity='mgr.smithi154.pplnxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T04:51:37.759 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:37 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:37.759 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:37 smithi152 ceph-mon[29073]: from='mgr.? 172.21.15.154:0/883741497' entity='mgr.smithi154.pplnxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi154.pplnxz/key"}]: dispatch 2024-04-02T04:51:37.759 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:37 smithi152 ceph-mon[29073]: from='mgr.? 172.21.15.154:0/883741497' entity='mgr.smithi154.pplnxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T04:51:37.759 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:37 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:37.922 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.919+0000 7fd700682700 1 -- 172.21.15.154:0/3550267504 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fd6f804bed0 con 0x7fd6e4003580 2024-04-02T04:51:37.923 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.920+0000 7fd6eeffd700 1 -- 172.21.15.154:0/3550267504 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 2 v2) v1 ==== 95+0+1021 (secure 0 0 0) 0x7fd6f4024410 con 0x7fd6e4003580 2024-04-02T04:51:37.923 INFO:teuthology.orchestra.run.smithi154.stdout: 2024-04-02T04:51:37.923 INFO:teuthology.orchestra.run.smithi154.stdout:{"epoch":2,"fsid":"2e0055d0-f0ac-11ee-b647-cb9ed24678a4","modified":"2024-04-02T04:51:32.406163Z","created":"2024-04-02T04:48:56.938040Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi152","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:3300","nonce":0},{"type":"v1","addr":"172.21.15.152:6789","nonce":0}]},"addr":"172.21.15.152:6789/0","public_addr":"172.21.15.152:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi154","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:3300","nonce":0},{"type":"v1","addr":"172.21.15.154:6789","nonce":0}]},"addr":"172.21.15.154:6789/0","public_addr":"172.21.15.154:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-02T04:51:37.925 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.922+0000 7fd700682700 1 -- 172.21.15.154:0/3550267504 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd6dc069360 msgr2=0x7fd6dc06b810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:37.926 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.922+0000 7fd700682700 1 --2- 172.21.15.154:0/3550267504 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd6dc069360 0x7fd6dc06b810 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 crypto rx=0x7fd6e8006fd0 tx=0x7fd6e8008040 comp rx=0 tx=0).stop 2024-04-02T04:51:37.926 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.922+0000 7fd700682700 1 -- 172.21.15.154:0/3550267504 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd6e4003580 msgr2=0x7fd6f8113330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:37.926 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.922+0000 7fd700682700 1 --2- 172.21.15.154:0/3550267504 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd6e4003580 0x7fd6f8113330 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 crypto rx=0x7fd6f4007c90 tx=0x7fd6f400b040 comp rx=0 tx=0).stop 2024-04-02T04:51:37.926 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.922+0000 7fd700682700 1 -- 172.21.15.154:0/3550267504 shutdown_connections 2024-04-02T04:51:37.926 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.922+0000 7fd700682700 1 --2- 172.21.15.154:0/3550267504 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd6dc069360 0x7fd6dc06b810 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:37.926 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.922+0000 7fd700682700 1 --2- 172.21.15.154:0/3550267504 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd6e4003580 0x7fd6f8113330 unknown :-1 s=CLOSED pgs=143 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:37.926 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.922+0000 7fd700682700 1 --2- 172.21.15.154:0/3550267504 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd6f8113870 0x7fd6f810d290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:37.926 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.922+0000 7fd700682700 1 -- 172.21.15.154:0/3550267504 >> 172.21.15.154:0/3550267504 conn(0x7fd6f80f75f0 msgr2=0x7fd6f80f9610 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:51:37.926 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.922+0000 7fd700682700 1 -- 172.21.15.154:0/3550267504 shutdown_connections 2024-04-02T04:51:37.926 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:51:37.922+0000 7fd700682700 1 -- 172.21.15.154:0/3550267504 wait complete. 2024-04-02T04:51:37.927 INFO:teuthology.orchestra.run.smithi154.stderr:dumped monmap epoch 2 2024-04-02T04:51:38.515 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-02T04:51:38.515 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph config generate-minimal-conf 2024-04-02T04:51:38.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:38 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:38.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:38 smithi154 ceph-mon[35729]: mgrmap e18: smithi152.jivdcl(active, since 27s), standbys: smithi154.pplnxz 2024-04-02T04:51:38.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:38 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:38.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:38 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mgr metadata", "who": "smithi154.pplnxz", "id": "smithi154.pplnxz"}]: dispatch 2024-04-02T04:51:38.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:38 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:38.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:38 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:51:38.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:38 smithi154 ceph-mon[35729]: from='client.? 172.21.15.154:0/3550267504' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:51:38.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:38 smithi154 ceph-mon[35729]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:38.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:38 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-04-02T04:51:38.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:38 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:38.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:38 smithi152 ceph-mon[29073]: mgrmap e18: smithi152.jivdcl(active, since 27s), standbys: smithi154.pplnxz 2024-04-02T04:51:38.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:38 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:38.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:38 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mgr metadata", "who": "smithi154.pplnxz", "id": "smithi154.pplnxz"}]: dispatch 2024-04-02T04:51:38.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:38 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:38.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:38 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:51:38.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:38 smithi152 ceph-mon[29073]: from='client.? 172.21.15.154:0/3550267504' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-02T04:51:38.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:38 smithi152 ceph-mon[29073]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:38.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:38 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-04-02T04:51:38.778 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:51:40.160 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.144+0000 7f9892683700 1 -- 172.21.15.152:0/1065710728 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f988c1019b0 msgr2=0x7f988c101d80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:40.160 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.144+0000 7f9892683700 1 --2- 172.21.15.152:0/1065710728 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f988c1019b0 0x7f988c101d80 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 crypto rx=0x7f9884009c60 tx=0x7f9884004d10 comp rx=0 tx=0).stop 2024-04-02T04:51:40.161 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.144+0000 7f9892683700 1 -- 172.21.15.152:0/1065710728 shutdown_connections 2024-04-02T04:51:40.161 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.144+0000 7f9892683700 1 --2- 172.21.15.152:0/1065710728 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f988c1019b0 0x7f988c101d80 unknown :-1 s=CLOSED pgs=144 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:40.161 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.144+0000 7f9892683700 1 -- 172.21.15.152:0/1065710728 >> 172.21.15.152:0/1065710728 conn(0x7f988c0f7510 msgr2=0x7f988c0f9920 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:51:40.161 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.144+0000 7f9892683700 1 -- 172.21.15.152:0/1065710728 shutdown_connections 2024-04-02T04:51:40.161 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.144+0000 7f9892683700 1 -- 172.21.15.152:0/1065710728 wait complete. 2024-04-02T04:51:40.161 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.145+0000 7f9892683700 1 Processor -- start 2024-04-02T04:51:40.161 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.145+0000 7f9892683700 1 -- start start 2024-04-02T04:51:40.161 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.145+0000 7f9892683700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f988c1019b0 0x7f988c1132a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:51:40.161 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.145+0000 7f9892683700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f988c1137e0 0x7f988c10d200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:51:40.161 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.145+0000 7f9892683700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f988c113df0 con 0x7f988c1019b0 2024-04-02T04:51:40.161 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.145+0000 7f9892683700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f988c10d740 con 0x7f988c1137e0 2024-04-02T04:51:40.161 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.145+0000 7f988bfff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f988c1019b0 0x7f988c1132a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:51:40.161 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.145+0000 7f988bfff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f988c1019b0 0x7f988c1132a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:34176/0 (socket says 172.21.15.152:34176) 2024-04-02T04:51:40.162 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.145+0000 7f988bfff700 1 -- 172.21.15.152:0/1043532692 learned_addr learned my addr 172.21.15.152:0/1043532692 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:51:40.162 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.145+0000 7f988b7fe700 1 --2- 172.21.15.152:0/1043532692 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f988c1137e0 0x7f988c10d200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:51:40.162 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.146+0000 7f988bfff700 1 -- 172.21.15.152:0/1043532692 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f988c1137e0 msgr2=0x7f988c10d200 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:40.162 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.146+0000 7f988bfff700 1 --2- 172.21.15.152:0/1043532692 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f988c1137e0 0x7f988c10d200 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:40.162 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.146+0000 7f988bfff700 1 -- 172.21.15.152:0/1043532692 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9884009940 con 0x7f988c1019b0 2024-04-02T04:51:40.162 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.146+0000 7f988b7fe700 1 --2- 172.21.15.152:0/1043532692 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f988c1137e0 0x7f988c10d200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-02T04:51:40.162 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.146+0000 7f988bfff700 1 --2- 172.21.15.152:0/1043532692 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f988c1019b0 0x7f988c1132a0 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f9884009910 tx=0x7f9884004d10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:51:40.162 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.146+0000 7f9888ff9700 1 -- 172.21.15.152:0/1043532692 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f988400e040 con 0x7f988c1019b0 2024-04-02T04:51:40.162 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.146+0000 7f9888ff9700 1 -- 172.21.15.152:0/1043532692 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f9884008e20 con 0x7f988c1019b0 2024-04-02T04:51:40.162 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.146+0000 7f9892683700 1 -- 172.21.15.152:0/1043532692 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f988c10d9c0 con 0x7f988c1019b0 2024-04-02T04:51:40.162 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.146+0000 7f9888ff9700 1 -- 172.21.15.152:0/1043532692 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f988401c7e0 con 0x7f988c1019b0 2024-04-02T04:51:40.162 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.146+0000 7f9892683700 1 -- 172.21.15.152:0/1043532692 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f988c10df80 con 0x7f988c1019b0 2024-04-02T04:51:40.162 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.147+0000 7f9892683700 1 -- 172.21.15.152:0/1043532692 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f988c0fd0b0 con 0x7f988c1019b0 2024-04-02T04:51:40.162 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.147+0000 7f9888ff9700 1 -- 172.21.15.152:0/1043532692 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 18) v1 ==== 86567+0+0 (secure 0 0 0) 0x7f9884012b70 con 0x7f988c1019b0 2024-04-02T04:51:40.162 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.148+0000 7f9888ff9700 1 --2- 172.21.15.152:0/1043532692 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f9874069370 0x7f987406b820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:51:40.163 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.148+0000 7f9888ff9700 1 -- 172.21.15.152:0/1043532692 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1366+0+0 (secure 0 0 0) 0x7f9884087f20 con 0x7f988c1019b0 2024-04-02T04:51:40.163 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.148+0000 7f988b7fe700 1 --2- 172.21.15.152:0/1043532692 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f9874069370 0x7f987406b820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:51:40.163 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.148+0000 7f988b7fe700 1 --2- 172.21.15.152:0/1043532692 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f9874069370 0x7f987406b820 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f9880007a80 tx=0x7f9880008040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:51:40.163 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.152+0000 7f9888ff9700 1 -- 172.21.15.152:0/1043532692 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f9884058ca0 con 0x7f988c1019b0 2024-04-02T04:51:40.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:40 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:51:40.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:40 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:40.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:40 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:40.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:40 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:40.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:40 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:51:40.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:40 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:51:40.422 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.418+0000 7f9892683700 1 -- 172.21.15.152:0/1043532692 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "config generate-minimal-conf"} v 0) v1 -- 0x7f988c05f5f0 con 0x7f988c1019b0 2024-04-02T04:51:40.422 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.418+0000 7f9888ff9700 1 -- 172.21.15.152:0/1043532692 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "config generate-minimal-conf"}]=0 v10) v1 ==== 76+0+227 (secure 0 0 0) 0x7f9884020100 con 0x7f988c1019b0 2024-04-02T04:51:40.422 INFO:teuthology.orchestra.run.smithi152.stdout:# minimal ceph.conf for 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 2024-04-02T04:51:40.422 INFO:teuthology.orchestra.run.smithi152.stdout:[global] 2024-04-02T04:51:40.422 INFO:teuthology.orchestra.run.smithi152.stdout: fsid = 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 2024-04-02T04:51:40.422 INFO:teuthology.orchestra.run.smithi152.stdout: mon_host = [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] 2024-04-02T04:51:40.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.422+0000 7f9892683700 1 -- 172.21.15.152:0/1043532692 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f9874069370 msgr2=0x7f987406b820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:40.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.422+0000 7f9892683700 1 --2- 172.21.15.152:0/1043532692 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f9874069370 0x7f987406b820 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f9880007a80 tx=0x7f9880008040 comp rx=0 tx=0).stop 2024-04-02T04:51:40.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.422+0000 7f9892683700 1 -- 172.21.15.152:0/1043532692 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f988c1019b0 msgr2=0x7f988c1132a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:40.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.422+0000 7f9892683700 1 --2- 172.21.15.152:0/1043532692 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f988c1019b0 0x7f988c1132a0 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 crypto rx=0x7f9884009910 tx=0x7f9884004d10 comp rx=0 tx=0).stop 2024-04-02T04:51:40.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.422+0000 7f9892683700 1 -- 172.21.15.152:0/1043532692 shutdown_connections 2024-04-02T04:51:40.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.422+0000 7f9892683700 1 --2- 172.21.15.152:0/1043532692 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f9874069370 0x7f987406b820 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:40.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.422+0000 7f9892683700 1 --2- 172.21.15.152:0/1043532692 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f988c1019b0 0x7f988c1132a0 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:40.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.422+0000 7f9892683700 1 --2- 172.21.15.152:0/1043532692 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f988c1137e0 0x7f988c10d200 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:40.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.422+0000 7f9892683700 1 -- 172.21.15.152:0/1043532692 >> 172.21.15.152:0/1043532692 conn(0x7f988c0f7510 msgr2=0x7f988c066160 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:51:40.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.422+0000 7f9892683700 1 -- 172.21.15.152:0/1043532692 shutdown_connections 2024-04-02T04:51:40.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:40.422+0000 7f9892683700 1 -- 172.21.15.152:0/1043532692 wait complete. 2024-04-02T04:51:40.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:40 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:51:40.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:40 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:40.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:40 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:40.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:40 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:40.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:40 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:51:40.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:40 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:51:41.048 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-02T04:51:41.049 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-02T04:51:41.049 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/etc/ceph/ceph.conf 2024-04-02T04:51:41.078 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-02T04:51:41.078 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:51:41.145 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-04-02T04:51:41.145 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd of=/etc/ceph/ceph.conf 2024-04-02T04:51:41.176 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-04-02T04:51:41.176 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-02T04:51:41.244 INFO:tasks.cephadm:Deploying OSDs... 2024-04-02T04:51:41.244 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-02T04:51:41.244 DEBUG:teuthology.orchestra.run.smithi152:> dd if=/scratch_devs of=/dev/stdout 2024-04-02T04:51:41.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:41 smithi152 ceph-mon[29073]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:41.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:41 smithi152 ceph-mon[29073]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-02T04:51:41.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:41 smithi152 ceph-mon[29073]: Updating smithi154:/etc/ceph/ceph.conf 2024-04-02T04:51:41.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:41 smithi152 ceph-mon[29073]: Updating smithi152:/var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T04:51:41.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:41 smithi152 ceph-mon[29073]: Updating smithi154:/var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T04:51:41.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:41 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/1043532692' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:51:41.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:41 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:41.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:41 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:41.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:41 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:41.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:41 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:41.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:41 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:41.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:41 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T04:51:41.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:41 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T04:51:41.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:41 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:51:41.260 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-02T04:51:41.260 DEBUG:teuthology.orchestra.run.smithi152:> stat /dev/vg_nvme/lv_1 2024-04-02T04:51:41.315 INFO:teuthology.orchestra.run.smithi152.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-02T04:51:41.315 INFO:teuthology.orchestra.run.smithi152.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T04:51:41.316 INFO:teuthology.orchestra.run.smithi152.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-04-02T04:51:41.316 INFO:teuthology.orchestra.run.smithi152.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T04:51:41.316 INFO:teuthology.orchestra.run.smithi152.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T04:51:41.316 INFO:teuthology.orchestra.run.smithi152.stdout:Access: 2024-04-02 04:51:01.751962674 +0000 2024-04-02T04:51:41.316 INFO:teuthology.orchestra.run.smithi152.stdout:Modify: 2024-04-02 04:45:20.250066481 +0000 2024-04-02T04:51:41.316 INFO:teuthology.orchestra.run.smithi152.stdout:Change: 2024-04-02 04:45:20.250066481 +0000 2024-04-02T04:51:41.316 INFO:teuthology.orchestra.run.smithi152.stdout: Birth: 2024-04-02 04:45:20.250066481 +0000 2024-04-02T04:51:41.316 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-02T04:51:41.384 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records in 2024-04-02T04:51:41.384 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records out 2024-04-02T04:51:41.384 INFO:teuthology.orchestra.run.smithi152.stderr:512 bytes copied, 0.000191418 s, 2.7 MB/s 2024-04-02T04:51:41.386 DEBUG:teuthology.orchestra.run.smithi152:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-02T04:51:41.442 DEBUG:teuthology.orchestra.run.smithi152:> stat /dev/vg_nvme/lv_2 2024-04-02T04:51:41.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:41 smithi154 ceph-mon[35729]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:41.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:41 smithi154 ceph-mon[35729]: Updating smithi152:/etc/ceph/ceph.conf 2024-04-02T04:51:41.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:41 smithi154 ceph-mon[35729]: Updating smithi154:/etc/ceph/ceph.conf 2024-04-02T04:51:41.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:41 smithi154 ceph-mon[35729]: Updating smithi152:/var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T04:51:41.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:41 smithi154 ceph-mon[35729]: Updating smithi154:/var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/config/ceph.conf 2024-04-02T04:51:41.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:41 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/1043532692' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:51:41.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:41 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:41.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:41 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:41.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:41 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:41.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:41 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:41.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:41 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:41.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:41 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T04:51:41.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:41 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T04:51:41.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:41 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:51:41.497 INFO:teuthology.orchestra.run.smithi152.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-02T04:51:41.497 INFO:teuthology.orchestra.run.smithi152.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T04:51:41.497 INFO:teuthology.orchestra.run.smithi152.stdout:Device: 5h/5d Inode: 684 Links: 1 2024-04-02T04:51:41.497 INFO:teuthology.orchestra.run.smithi152.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T04:51:41.497 INFO:teuthology.orchestra.run.smithi152.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T04:51:41.497 INFO:teuthology.orchestra.run.smithi152.stdout:Access: 2024-04-02 04:51:01.752962661 +0000 2024-04-02T04:51:41.497 INFO:teuthology.orchestra.run.smithi152.stdout:Modify: 2024-04-02 04:45:20.589062274 +0000 2024-04-02T04:51:41.497 INFO:teuthology.orchestra.run.smithi152.stdout:Change: 2024-04-02 04:45:20.589062274 +0000 2024-04-02T04:51:41.497 INFO:teuthology.orchestra.run.smithi152.stdout: Birth: 2024-04-02 04:45:20.589062274 +0000 2024-04-02T04:51:41.498 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-02T04:51:41.564 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records in 2024-04-02T04:51:41.564 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records out 2024-04-02T04:51:41.564 INFO:teuthology.orchestra.run.smithi152.stderr:512 bytes copied, 0.000169711 s, 3.0 MB/s 2024-04-02T04:51:41.566 DEBUG:teuthology.orchestra.run.smithi152:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-02T04:51:41.621 DEBUG:teuthology.orchestra.run.smithi152:> stat /dev/vg_nvme/lv_3 2024-04-02T04:51:41.676 INFO:teuthology.orchestra.run.smithi152.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-02T04:51:41.677 INFO:teuthology.orchestra.run.smithi152.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T04:51:41.677 INFO:teuthology.orchestra.run.smithi152.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-04-02T04:51:41.677 INFO:teuthology.orchestra.run.smithi152.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T04:51:41.677 INFO:teuthology.orchestra.run.smithi152.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T04:51:41.677 INFO:teuthology.orchestra.run.smithi152.stdout:Access: 2024-04-02 04:51:01.752962661 +0000 2024-04-02T04:51:41.677 INFO:teuthology.orchestra.run.smithi152.stdout:Modify: 2024-04-02 04:45:20.943057881 +0000 2024-04-02T04:51:41.677 INFO:teuthology.orchestra.run.smithi152.stdout:Change: 2024-04-02 04:45:20.943057881 +0000 2024-04-02T04:51:41.677 INFO:teuthology.orchestra.run.smithi152.stdout: Birth: 2024-04-02 04:45:20.943057881 +0000 2024-04-02T04:51:41.677 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-02T04:51:41.743 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records in 2024-04-02T04:51:41.744 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records out 2024-04-02T04:51:41.744 INFO:teuthology.orchestra.run.smithi152.stderr:512 bytes copied, 0.000194577 s, 2.6 MB/s 2024-04-02T04:51:41.745 DEBUG:teuthology.orchestra.run.smithi152:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-02T04:51:41.801 DEBUG:teuthology.orchestra.run.smithi152:> stat /dev/vg_nvme/lv_4 2024-04-02T04:51:41.855 INFO:teuthology.orchestra.run.smithi152.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-02T04:51:41.855 INFO:teuthology.orchestra.run.smithi152.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T04:51:41.855 INFO:teuthology.orchestra.run.smithi152.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-04-02T04:51:41.855 INFO:teuthology.orchestra.run.smithi152.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T04:51:41.856 INFO:teuthology.orchestra.run.smithi152.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T04:51:41.856 INFO:teuthology.orchestra.run.smithi152.stdout:Access: 2024-04-02 04:51:01.753962649 +0000 2024-04-02T04:51:41.856 INFO:teuthology.orchestra.run.smithi152.stdout:Modify: 2024-04-02 04:45:21.286053624 +0000 2024-04-02T04:51:41.856 INFO:teuthology.orchestra.run.smithi152.stdout:Change: 2024-04-02 04:45:21.286053624 +0000 2024-04-02T04:51:41.856 INFO:teuthology.orchestra.run.smithi152.stdout: Birth: 2024-04-02 04:45:21.286053624 +0000 2024-04-02T04:51:41.856 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-02T04:51:41.920 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records in 2024-04-02T04:51:41.920 INFO:teuthology.orchestra.run.smithi152.stderr:1+0 records out 2024-04-02T04:51:41.920 INFO:teuthology.orchestra.run.smithi152.stderr:512 bytes copied, 0.000193481 s, 2.6 MB/s 2024-04-02T04:51:41.922 DEBUG:teuthology.orchestra.run.smithi152:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-02T04:51:41.977 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-04-02T04:51:41.977 DEBUG:teuthology.orchestra.run.smithi154:> dd if=/scratch_devs of=/dev/stdout 2024-04-02T04:51:41.996 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-02T04:51:41.996 DEBUG:teuthology.orchestra.run.smithi154:> stat /dev/vg_nvme/lv_1 2024-04-02T04:51:42.054 INFO:teuthology.orchestra.run.smithi154.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-02T04:51:42.054 INFO:teuthology.orchestra.run.smithi154.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T04:51:42.054 INFO:teuthology.orchestra.run.smithi154.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-04-02T04:51:42.054 INFO:teuthology.orchestra.run.smithi154.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T04:51:42.054 INFO:teuthology.orchestra.run.smithi154.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T04:51:42.054 INFO:teuthology.orchestra.run.smithi154.stdout:Access: 2024-04-02 04:51:31.465483109 +0000 2024-04-02T04:51:42.054 INFO:teuthology.orchestra.run.smithi154.stdout:Modify: 2024-04-02 04:45:26.090869179 +0000 2024-04-02T04:51:42.054 INFO:teuthology.orchestra.run.smithi154.stdout:Change: 2024-04-02 04:45:26.090869179 +0000 2024-04-02T04:51:42.054 INFO:teuthology.orchestra.run.smithi154.stdout: Birth: 2024-04-02 04:45:26.090869179 +0000 2024-04-02T04:51:42.055 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-02T04:51:42.124 INFO:teuthology.orchestra.run.smithi154.stderr:1+0 records in 2024-04-02T04:51:42.124 INFO:teuthology.orchestra.run.smithi154.stderr:1+0 records out 2024-04-02T04:51:42.124 INFO:teuthology.orchestra.run.smithi154.stderr:512 bytes copied, 0.000286327 s, 1.8 MB/s 2024-04-02T04:51:42.125 DEBUG:teuthology.orchestra.run.smithi154:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-02T04:51:42.182 DEBUG:teuthology.orchestra.run.smithi154:> stat /dev/vg_nvme/lv_2 2024-04-02T04:51:42.239 INFO:teuthology.orchestra.run.smithi154.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-02T04:51:42.239 INFO:teuthology.orchestra.run.smithi154.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T04:51:42.239 INFO:teuthology.orchestra.run.smithi154.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-04-02T04:51:42.240 INFO:teuthology.orchestra.run.smithi154.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T04:51:42.240 INFO:teuthology.orchestra.run.smithi154.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T04:51:42.240 INFO:teuthology.orchestra.run.smithi154.stdout:Access: 2024-04-02 04:51:31.465483109 +0000 2024-04-02T04:51:42.240 INFO:teuthology.orchestra.run.smithi154.stdout:Modify: 2024-04-02 04:45:26.425860634 +0000 2024-04-02T04:51:42.240 INFO:teuthology.orchestra.run.smithi154.stdout:Change: 2024-04-02 04:45:26.425860634 +0000 2024-04-02T04:51:42.240 INFO:teuthology.orchestra.run.smithi154.stdout: Birth: 2024-04-02 04:45:26.425860634 +0000 2024-04-02T04:51:42.240 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-02T04:51:42.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:42 smithi152 ceph-mon[29073]: Reconfiguring mon.smithi152 (unknown last config time)... 2024-04-02T04:51:42.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:42 smithi152 ceph-mon[29073]: Reconfiguring daemon mon.smithi152 on smithi152 2024-04-02T04:51:42.307 INFO:teuthology.orchestra.run.smithi154.stderr:1+0 records in 2024-04-02T04:51:42.307 INFO:teuthology.orchestra.run.smithi154.stderr:1+0 records out 2024-04-02T04:51:42.307 INFO:teuthology.orchestra.run.smithi154.stderr:512 bytes copied, 0.000231218 s, 2.2 MB/s 2024-04-02T04:51:42.309 DEBUG:teuthology.orchestra.run.smithi154:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-02T04:51:42.366 DEBUG:teuthology.orchestra.run.smithi154:> stat /dev/vg_nvme/lv_3 2024-04-02T04:51:42.423 INFO:teuthology.orchestra.run.smithi154.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-02T04:51:42.423 INFO:teuthology.orchestra.run.smithi154.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T04:51:42.423 INFO:teuthology.orchestra.run.smithi154.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-04-02T04:51:42.424 INFO:teuthology.orchestra.run.smithi154.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T04:51:42.424 INFO:teuthology.orchestra.run.smithi154.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T04:51:42.424 INFO:teuthology.orchestra.run.smithi154.stdout:Access: 2024-04-02 04:51:31.466483084 +0000 2024-04-02T04:51:42.424 INFO:teuthology.orchestra.run.smithi154.stdout:Modify: 2024-04-02 04:45:26.765851962 +0000 2024-04-02T04:51:42.424 INFO:teuthology.orchestra.run.smithi154.stdout:Change: 2024-04-02 04:45:26.765851962 +0000 2024-04-02T04:51:42.424 INFO:teuthology.orchestra.run.smithi154.stdout: Birth: 2024-04-02 04:45:26.765851962 +0000 2024-04-02T04:51:42.424 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-02T04:51:42.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:42 smithi154 ceph-mon[35729]: Reconfiguring mon.smithi152 (unknown last config time)... 2024-04-02T04:51:42.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:42 smithi154 ceph-mon[35729]: Reconfiguring daemon mon.smithi152 on smithi152 2024-04-02T04:51:42.490 INFO:teuthology.orchestra.run.smithi154.stderr:1+0 records in 2024-04-02T04:51:42.490 INFO:teuthology.orchestra.run.smithi154.stderr:1+0 records out 2024-04-02T04:51:42.490 INFO:teuthology.orchestra.run.smithi154.stderr:512 bytes copied, 0.000208753 s, 2.5 MB/s 2024-04-02T04:51:42.491 DEBUG:teuthology.orchestra.run.smithi154:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-02T04:51:42.548 DEBUG:teuthology.orchestra.run.smithi154:> stat /dev/vg_nvme/lv_4 2024-04-02T04:51:42.604 INFO:teuthology.orchestra.run.smithi154.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-02T04:51:42.604 INFO:teuthology.orchestra.run.smithi154.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-02T04:51:42.604 INFO:teuthology.orchestra.run.smithi154.stdout:Device: 5h/5d Inode: 724 Links: 1 2024-04-02T04:51:42.604 INFO:teuthology.orchestra.run.smithi154.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T04:51:42.604 INFO:teuthology.orchestra.run.smithi154.stdout:Context: system_u:object_r:device_t:s0 2024-04-02T04:51:42.605 INFO:teuthology.orchestra.run.smithi154.stdout:Access: 2024-04-02 04:51:31.466483084 +0000 2024-04-02T04:51:42.605 INFO:teuthology.orchestra.run.smithi154.stdout:Modify: 2024-04-02 04:45:27.106843264 +0000 2024-04-02T04:51:42.605 INFO:teuthology.orchestra.run.smithi154.stdout:Change: 2024-04-02 04:45:27.106843264 +0000 2024-04-02T04:51:42.605 INFO:teuthology.orchestra.run.smithi154.stdout: Birth: 2024-04-02 04:45:27.106843264 +0000 2024-04-02T04:51:42.605 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-02T04:51:42.670 INFO:teuthology.orchestra.run.smithi154.stderr:1+0 records in 2024-04-02T04:51:42.670 INFO:teuthology.orchestra.run.smithi154.stderr:1+0 records out 2024-04-02T04:51:42.670 INFO:teuthology.orchestra.run.smithi154.stderr:512 bytes copied, 0.000205343 s, 2.5 MB/s 2024-04-02T04:51:42.672 DEBUG:teuthology.orchestra.run.smithi154:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-02T04:51:42.728 INFO:tasks.cephadm:Deploying osd.0 on smithi152 with /dev/vg_nvme/lv_4... 2024-04-02T04:51:42.728 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-02T04:51:43.415 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:43 smithi152 ceph-mon[29073]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:43.415 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:43 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:43.415 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:43 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:43.415 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:43 smithi152 ceph-mon[29073]: Reconfiguring mgr.smithi152.jivdcl (unknown last config time)... 2024-04-02T04:51:43.415 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:43 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi152.jivdcl", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T04:51:43.415 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:43 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T04:51:43.415 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:43 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:51:43.415 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:43 smithi152 ceph-mon[29073]: Reconfiguring daemon mgr.smithi152.jivdcl on smithi152 2024-04-02T04:51:43.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:43 smithi154 ceph-mon[35729]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:43.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:43 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:43.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:43 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:43.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:43 smithi154 ceph-mon[35729]: Reconfiguring mgr.smithi152.jivdcl (unknown last config time)... 2024-04-02T04:51:43.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:43 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi152.jivdcl", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T04:51:43.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:43 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T04:51:43.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:43 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:51:43.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:43 smithi154 ceph-mon[35729]: Reconfiguring daemon mgr.smithi152.jivdcl on smithi152 2024-04-02T04:51:43.515 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:51:44.548 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:44 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:44.548 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:44 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:44.549 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:44 smithi152 ceph-mon[29073]: Reconfiguring crash.smithi152 (monmap changed)... 2024-04-02T04:51:44.549 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:44 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi152", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-02T04:51:44.549 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:44 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:51:44.549 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:44 smithi152 ceph-mon[29073]: Reconfiguring daemon crash.smithi152 on smithi152 2024-04-02T04:51:44.549 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:44 smithi152 ceph-mon[29073]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:44.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:44 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:44.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:44 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:44.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:44 smithi154 ceph-mon[35729]: Reconfiguring crash.smithi152 (monmap changed)... 2024-04-02T04:51:44.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:44 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi152", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-02T04:51:44.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:44 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:51:44.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:44 smithi154 ceph-mon[35729]: Reconfiguring daemon crash.smithi152 on smithi152 2024-04-02T04:51:44.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:44 smithi154 ceph-mon[35729]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:46.479 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:46 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:46.479 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:46 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:46.479 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:46 smithi152 ceph-mon[29073]: Reconfiguring alertmanager.smithi152 (dependencies changed)... 2024-04-02T04:51:46.479 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:46 smithi152 ceph-mon[29073]: Reconfiguring daemon alertmanager.smithi152 on smithi152 2024-04-02T04:51:46.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:46 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:46.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:46 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:46.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:46 smithi154 ceph-mon[35729]: Reconfiguring alertmanager.smithi152 (dependencies changed)... 2024-04-02T04:51:46.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:46 smithi154 ceph-mon[35729]: Reconfiguring daemon alertmanager.smithi152 on smithi152 2024-04-02T04:51:46.721 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-02T04:51:46.722 INFO:teuthology.orchestra.run.smithi152.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-02T04:51:46.722 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10+0 records in 2024-04-02T04:51:46.722 INFO:teuthology.orchestra.run.smithi152.stderr:10+0 records out 2024-04-02T04:51:46.722 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0114886 s, 913 MB/s 2024-04-02T04:51:46.722 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping successful for: 2024-04-02T04:51:47.300 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:47 smithi152 ceph-mon[29073]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:47.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:47 smithi154 ceph-mon[35729]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:49.076 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi152:vg_nvme/lv_4 2024-04-02T04:51:49.361 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:49 smithi152 ceph-mon[29073]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:49.401 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:51:49.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:49 smithi154 ceph-mon[35729]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:51.423 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:51 smithi152 ceph-mon[29073]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:51.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:51 smithi154 ceph-mon[35729]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:51.633 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.628+0000 7fb0b054d700 1 -- 172.21.15.152:0/3027920842 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb0a80fce70 msgr2=0x7fb0a80fd280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:51.633 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.628+0000 7fb0b054d700 1 --2- 172.21.15.152:0/3027920842 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb0a80fce70 0x7fb0a80fd280 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 crypto rx=0x7fb0a4009ce0 tx=0x7fb0a4004d10 comp rx=0 tx=0).stop 2024-04-02T04:51:51.633 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.628+0000 7fb0b054d700 1 -- 172.21.15.152:0/3027920842 shutdown_connections 2024-04-02T04:51:51.634 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.628+0000 7fb0b054d700 1 --2- 172.21.15.152:0/3027920842 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb0a80fdfe0 0x7fb0a80fe450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:51.634 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.628+0000 7fb0b054d700 1 --2- 172.21.15.152:0/3027920842 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb0a80fce70 0x7fb0a80fd280 unknown :-1 s=CLOSED pgs=146 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:51.634 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.628+0000 7fb0b054d700 1 -- 172.21.15.152:0/3027920842 >> 172.21.15.152:0/3027920842 conn(0x7fb0a80f8550 msgr2=0x7fb0a80fa960 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:51:51.634 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.628+0000 7fb0b054d700 1 -- 172.21.15.152:0/3027920842 shutdown_connections 2024-04-02T04:51:51.634 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.628+0000 7fb0b054d700 1 -- 172.21.15.152:0/3027920842 wait complete. 2024-04-02T04:51:51.634 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.629+0000 7fb0b054d700 1 Processor -- start 2024-04-02T04:51:51.634 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.629+0000 7fb0b054d700 1 -- start start 2024-04-02T04:51:51.634 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.629+0000 7fb0b054d700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb0a80fce70 0x7fb0a810da90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:51:51.634 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.629+0000 7fb0b054d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb0a80fdfe0 0x7fb0a810dfd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:51:51.634 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.629+0000 7fb0b054d700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb0a810e660 con 0x7fb0a80fdfe0 2024-04-02T04:51:51.634 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.629+0000 7fb0b054d700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb0a8113000 con 0x7fb0a80fce70 2024-04-02T04:51:51.634 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.629+0000 7fb0adae8700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb0a80fdfe0 0x7fb0a810dfd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:51:51.635 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.629+0000 7fb0adae8700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb0a80fdfe0 0x7fb0a810dfd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:49868/0 (socket says 172.21.15.152:49868) 2024-04-02T04:51:51.635 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.629+0000 7fb0adae8700 1 -- 172.21.15.152:0/238862551 learned_addr learned my addr 172.21.15.152:0/238862551 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:51:51.635 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.629+0000 7fb0ae2e9700 1 --2- 172.21.15.152:0/238862551 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb0a80fce70 0x7fb0a810da90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:51:51.635 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.629+0000 7fb0adae8700 1 -- 172.21.15.152:0/238862551 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb0a80fce70 msgr2=0x7fb0a810da90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:51:51.635 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.629+0000 7fb0adae8700 1 --2- 172.21.15.152:0/238862551 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb0a80fce70 0x7fb0a810da90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:51:51.635 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.629+0000 7fb0adae8700 1 -- 172.21.15.152:0/238862551 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb0a4009940 con 0x7fb0a80fdfe0 2024-04-02T04:51:51.635 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.629+0000 7fb0adae8700 1 --2- 172.21.15.152:0/238862551 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb0a80fdfe0 0x7fb0a810dfd0 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7fb09800b790 tx=0x7fb098009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:51:51.635 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.629+0000 7fb09effd700 1 -- 172.21.15.152:0/238862551 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fb09800f770 con 0x7fb0a80fdfe0 2024-04-02T04:51:51.635 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.630+0000 7fb09effd700 1 -- 172.21.15.152:0/238862551 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fb09800fda0 con 0x7fb0a80fdfe0 2024-04-02T04:51:51.635 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.630+0000 7fb0b054d700 1 -- 172.21.15.152:0/238862551 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb0a8113260 con 0x7fb0a80fdfe0 2024-04-02T04:51:51.635 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.630+0000 7fb09effd700 1 -- 172.21.15.152:0/238862551 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fb09800d310 con 0x7fb0a80fdfe0 2024-04-02T04:51:51.635 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.630+0000 7fb0b054d700 1 -- 172.21.15.152:0/238862551 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb0a81137d0 con 0x7fb0a80fdfe0 2024-04-02T04:51:51.636 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.630+0000 7fb0b054d700 1 -- 172.21.15.152:0/238862551 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb0a804c710 con 0x7fb0a80fdfe0 2024-04-02T04:51:51.636 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.631+0000 7fb09effd700 1 -- 172.21.15.152:0/238862551 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 18) v1 ==== 86567+0+0 (secure 0 0 0) 0x7fb098010420 con 0x7fb0a80fdfe0 2024-04-02T04:51:51.636 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.631+0000 7fb09effd700 1 --2- 172.21.15.152:0/238862551 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb094069210 0x7fb09406b6c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:51:51.636 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.632+0000 7fb0ae2e9700 1 --2- 172.21.15.152:0/238862551 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb094069210 0x7fb09406b6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:51:51.637 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.632+0000 7fb09effd700 1 -- 172.21.15.152:0/238862551 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1366+0+0 (secure 0 0 0) 0x7fb0980860a0 con 0x7fb0a80fdfe0 2024-04-02T04:51:51.637 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.632+0000 7fb0ae2e9700 1 --2- 172.21.15.152:0/238862551 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb094069210 0x7fb09406b6c0 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7fb0a40095a0 tx=0x7fb0a4004d10 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:51:51.639 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.636+0000 7fb09effd700 1 -- 172.21.15.152:0/238862551 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fb098056f40 con 0x7fb0a80fdfe0 2024-04-02T04:51:51.891 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:51:51.887+0000 7fb0b054d700 1 -- 172.21.15.152:0/238862551 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_4", "target": ["mon-mgr", ""]}) v1 -- 0x7fb0a81022b0 con 0x7fb094069210 2024-04-02T04:51:52.326 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:52 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:51:52.326 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:52 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:51:52.326 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:52 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:51:52.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:52 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:51:52.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:52 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:51:52.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:52 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:51:53.385 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:53 smithi152 ceph-mon[29073]: from='client.14242 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:51:53.385 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:53 smithi152 ceph-mon[29073]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:53.385 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:53 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:53.385 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:53 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:53.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:53 smithi154 ceph-mon[35729]: from='client.14242 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:51:53.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:53 smithi154 ceph-mon[35729]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:53.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:53 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:53.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:53 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:54.469 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:54 smithi152 ceph-mon[29073]: Reconfiguring grafana.smithi152 (dependencies changed)... 2024-04-02T04:51:54.470 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:54 smithi152 ceph-mon[29073]: Reconfiguring daemon grafana.smithi152 on smithi152 2024-04-02T04:51:54.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:54 smithi154 ceph-mon[35729]: Reconfiguring grafana.smithi152 (dependencies changed)... 2024-04-02T04:51:54.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:54 smithi154 ceph-mon[35729]: Reconfiguring daemon grafana.smithi152 on smithi152 2024-04-02T04:51:55.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:55 smithi154 ceph-mon[35729]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:55.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:55 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:51:55.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:55 smithi152 ceph-mon[29073]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:55.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:55 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:51:57.347 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:57 smithi152 ceph-mon[29073]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:57.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:57 smithi154 ceph-mon[35729]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:58.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:58 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:58.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:58 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:58.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:58 smithi154 ceph-mon[35729]: Reconfiguring prometheus.smithi152 (dependencies changed)... 2024-04-02T04:51:58.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:58 smithi154 ceph-mon[35729]: Reconfiguring daemon prometheus.smithi152 on smithi152 2024-04-02T04:51:58.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:58 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:58.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:58 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:51:58.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:58 smithi152 ceph-mon[29073]: Reconfiguring prometheus.smithi152 (dependencies changed)... 2024-04-02T04:51:58.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:58 smithi152 ceph-mon[29073]: Reconfiguring daemon prometheus.smithi152 on smithi152 2024-04-02T04:51:59.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:59 smithi154 ceph-mon[35729]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:59.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:59 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/1002238841' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3f25c359-f643-4dfb-ba7b-30bb1a2dfcb2"}]: dispatch 2024-04-02T04:51:59.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:59 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/1002238841' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3f25c359-f643-4dfb-ba7b-30bb1a2dfcb2"}]': finished 2024-04-02T04:51:59.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:59 smithi154 ceph-mon[35729]: osdmap e6: 1 total, 0 up, 1 in 2024-04-02T04:51:59.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:59 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:51:59.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:51:59 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/603674198' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:51:59.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:59 smithi152 ceph-mon[29073]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:51:59.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:59 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/1002238841' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3f25c359-f643-4dfb-ba7b-30bb1a2dfcb2"}]: dispatch 2024-04-02T04:51:59.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:59 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/1002238841' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3f25c359-f643-4dfb-ba7b-30bb1a2dfcb2"}]': finished 2024-04-02T04:51:59.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:59 smithi152 ceph-mon[29073]: osdmap e6: 1 total, 0 up, 1 in 2024-04-02T04:51:59.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:59 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:51:59.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:51:59 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/603674198' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:52:01.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:01 smithi154 ceph-mon[35729]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:52:01.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:01 smithi152 ceph-mon[29073]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:52:03.568 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:03 smithi152 ceph-mon[29073]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:52:03.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:03 smithi154 ceph-mon[35729]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:52:05.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:05 smithi154 ceph-mon[35729]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:52:05.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:05 smithi152 ceph-mon[29073]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:52:07.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:07 smithi152 ceph-mon[29073]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:52:07.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:07 smithi154 ceph-mon[35729]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:52:09.448 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:09 smithi152 ceph-mon[29073]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:52:09.448 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:09 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:09.448 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:09 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:09.448 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:09 smithi152 ceph-mon[29073]: Reconfiguring crash.smithi154 (monmap changed)... 2024-04-02T04:52:09.448 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:09 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi154", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-02T04:52:09.448 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:09 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:52:09.449 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:09 smithi152 ceph-mon[29073]: Reconfiguring daemon crash.smithi154 on smithi154 2024-04-02T04:52:09.539 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:09 smithi154 ceph-mon[35729]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:52:09.539 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:09 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:09.539 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:09 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:09.539 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:09 smithi154 ceph-mon[35729]: Reconfiguring crash.smithi154 (monmap changed)... 2024-04-02T04:52:09.539 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:09 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi154", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-02T04:52:09.539 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:09 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:52:09.539 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:09 smithi154 ceph-mon[35729]: Reconfiguring daemon crash.smithi154 on smithi154 2024-04-02T04:52:10.684 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:10 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-02T04:52:10.685 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:10 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:52:10.685 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:10 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:10.685 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:10 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:10.685 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:10 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi154.pplnxz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T04:52:10.685 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:10 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T04:52:10.685 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:10 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:52:10.685 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:10 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:52:10.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:10 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-02T04:52:10.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:10 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:52:10.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:10 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:10.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:10 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:10.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:10 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi154.pplnxz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T04:52:10.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:10 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T04:52:10.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:10 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:52:10.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:10 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:52:11.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:11 smithi152 ceph-mon[29073]: Deploying daemon osd.0 on smithi152 2024-04-02T04:52:11.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:11 smithi152 ceph-mon[29073]: Reconfiguring mgr.smithi154.pplnxz (monmap changed)... 2024-04-02T04:52:11.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:11 smithi152 ceph-mon[29073]: Reconfiguring daemon mgr.smithi154.pplnxz on smithi154 2024-04-02T04:52:11.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:11 smithi152 ceph-mon[29073]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:52:11.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:11 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:11.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:11 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:11.508 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:11 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T04:52:11.508 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:11 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T04:52:11.508 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:11 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:52:11.573 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:11 smithi154 ceph-mon[35729]: Deploying daemon osd.0 on smithi152 2024-04-02T04:52:11.573 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:11 smithi154 ceph-mon[35729]: Reconfiguring mgr.smithi154.pplnxz (monmap changed)... 2024-04-02T04:52:11.573 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:11 smithi154 ceph-mon[35729]: Reconfiguring daemon mgr.smithi154.pplnxz on smithi154 2024-04-02T04:52:11.573 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:11 smithi154 ceph-mon[35729]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:52:11.573 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:11 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:11.573 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:11 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:11.574 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:11 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-02T04:52:11.574 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:11 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-02T04:52:11.574 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:11 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:52:12.352 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:12 smithi152 ceph-mon[29073]: Reconfiguring mon.smithi154 (monmap changed)... 2024-04-02T04:52:12.352 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:12 smithi152 ceph-mon[29073]: Reconfiguring daemon mon.smithi154 on smithi154 2024-04-02T04:52:12.352 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:12 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:12.352 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:12 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:12.353 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:12 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T04:52:12.353 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:12 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi152.front.sepia.ceph.com:9093"}]: dispatch 2024-04-02T04:52:12.353 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:12 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:12.353 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:12 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T04:52:12.353 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:12 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152.front.sepia.ceph.com:3000"}]: dispatch 2024-04-02T04:52:12.353 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:12 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:12.353 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:12 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T04:52:12.353 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:12 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2024-04-02T04:52:12.353 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:12 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:12.353 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:12 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:52:12.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:12 smithi154 ceph-mon[35729]: Reconfiguring mon.smithi154 (monmap changed)... 2024-04-02T04:52:12.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:12 smithi154 ceph-mon[35729]: Reconfiguring daemon mon.smithi154 on smithi154 2024-04-02T04:52:12.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:12 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:12.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:12 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:12.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:12 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T04:52:12.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:12 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi152.front.sepia.ceph.com:9093"}]: dispatch 2024-04-02T04:52:12.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:12 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:12.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:12 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T04:52:12.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:12 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152.front.sepia.ceph.com:3000"}]: dispatch 2024-04-02T04:52:12.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:12 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:12.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:12 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T04:52:12.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:12 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2024-04-02T04:52:12.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:12 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:12.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:12 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:52:13.561 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:13 smithi154 ceph-mon[35729]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:52:13.561 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:13 smithi154 ceph-mon[35729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T04:52:13.561 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:13 smithi154 ceph-mon[35729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi152.front.sepia.ceph.com:9093"}]: dispatch 2024-04-02T04:52:13.562 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:13 smithi154 ceph-mon[35729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T04:52:13.562 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:13 smithi154 ceph-mon[35729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152.front.sepia.ceph.com:3000"}]: dispatch 2024-04-02T04:52:13.562 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:13 smithi154 ceph-mon[35729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T04:52:13.562 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:13 smithi154 ceph-mon[35729]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2024-04-02T04:52:13.570 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:13 smithi152 ceph-mon[29073]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:52:13.570 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:13 smithi152 ceph-mon[29073]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-02T04:52:13.570 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:13 smithi152 ceph-mon[29073]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi152.front.sepia.ceph.com:9093"}]: dispatch 2024-04-02T04:52:13.571 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:13 smithi152 ceph-mon[29073]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-02T04:52:13.571 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:13 smithi152 ceph-mon[29073]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi152.front.sepia.ceph.com:3000"}]: dispatch 2024-04-02T04:52:13.571 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:13 smithi152 ceph-mon[29073]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-02T04:52:13.571 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:13 smithi152 ceph-mon[29073]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi152.front.sepia.ceph.com:9095"}]: dispatch 2024-04-02T04:52:15.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:14 smithi154 ceph-mon[35729]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:52:15.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:14 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:15.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:14 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:15.254 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:14 smithi152 ceph-mon[29073]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:52:15.254 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:14 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:15.254 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:14 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:16.813 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:16 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:16.813 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:16 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:16.813 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:16 smithi152 ceph-mon[29073]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:52:16.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:16 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:16.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:16 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:16.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:16 smithi154 ceph-mon[35729]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:52:19.171 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:19 smithi152 ceph-mon[29073]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:52:19.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:19 smithi154 ceph-mon[35729]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:52:20.208 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:20 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:20.208 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:20 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:20.208 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:20 smithi152 ceph-mon[29073]: from='osd.0 [v2:172.21.15.152:6802/718734380,v1:172.21.15.152:6803/718734380]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-02T04:52:20.208 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:20 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:52:20.208 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:20 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:52:20.208 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:20 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:20.208 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:20 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:52:20.268 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:20.264+0000 7fb09effd700 1 -- 172.21.15.152:0/238862551 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7fb0a81022b0 con 0x7fb094069210 2024-04-02T04:52:20.268 INFO:teuthology.orchestra.run.smithi152.stdout:Created osd(s) 0 on host 'smithi152' 2024-04-02T04:52:20.270 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:20.265+0000 7fb0b054d700 1 -- 172.21.15.152:0/238862551 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb094069210 msgr2=0x7fb09406b6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:52:20.270 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:20.265+0000 7fb0b054d700 1 --2- 172.21.15.152:0/238862551 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb094069210 0x7fb09406b6c0 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 crypto rx=0x7fb0a40095a0 tx=0x7fb0a4004d10 comp rx=0 tx=0).stop 2024-04-02T04:52:20.271 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:20.266+0000 7fb0b054d700 1 -- 172.21.15.152:0/238862551 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb0a80fdfe0 msgr2=0x7fb0a810dfd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:52:20.271 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:20.266+0000 7fb0b054d700 1 --2- 172.21.15.152:0/238862551 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb0a80fdfe0 0x7fb0a810dfd0 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 crypto rx=0x7fb09800b790 tx=0x7fb098009250 comp rx=0 tx=0).stop 2024-04-02T04:52:20.271 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:20.266+0000 7fb0b054d700 1 -- 172.21.15.152:0/238862551 shutdown_connections 2024-04-02T04:52:20.271 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:20.266+0000 7fb0b054d700 1 --2- 172.21.15.152:0/238862551 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb094069210 0x7fb09406b6c0 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:52:20.271 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:20.266+0000 7fb0b054d700 1 --2- 172.21.15.152:0/238862551 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb0a80fce70 0x7fb0a810da90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:52:20.271 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:20.266+0000 7fb0b054d700 1 --2- 172.21.15.152:0/238862551 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb0a80fdfe0 0x7fb0a810dfd0 unknown :-1 s=CLOSED pgs=147 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:52:20.271 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:20.266+0000 7fb0b054d700 1 -- 172.21.15.152:0/238862551 >> 172.21.15.152:0/238862551 conn(0x7fb0a80f8550 msgr2=0x7fb0a80f9b70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:52:20.271 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:20.266+0000 7fb0b054d700 1 -- 172.21.15.152:0/238862551 shutdown_connections 2024-04-02T04:52:20.271 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:20.266+0000 7fb0b054d700 1 -- 172.21.15.152:0/238862551 wait complete. 2024-04-02T04:52:20.358 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:20 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:20.358 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:20 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:20.359 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:20 smithi154 ceph-mon[35729]: from='osd.0 [v2:172.21.15.152:6802/718734380,v1:172.21.15.152:6803/718734380]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-02T04:52:20.359 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:20 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:52:20.359 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:20 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:52:20.359 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:20 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:20.359 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:20 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:52:20.833 DEBUG:teuthology.orchestra.run.smithi152:osd.0> sudo journalctl -f -n 0 -u ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4@osd.0.service 2024-04-02T04:52:20.835 INFO:tasks.cephadm:Deploying osd.1 on smithi152 with /dev/vg_nvme/lv_3... 2024-04-02T04:52:20.836 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-02T04:52:21.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:21 smithi154 ceph-mon[35729]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:52:21.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:21 smithi154 ceph-mon[35729]: from='osd.0 [v2:172.21.15.152:6802/718734380,v1:172.21.15.152:6803/718734380]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-02T04:52:21.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:21 smithi154 ceph-mon[35729]: osdmap e7: 1 total, 0 up, 1 in 2024-04-02T04:52:21.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:21 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:52:21.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:21 smithi154 ceph-mon[35729]: from='osd.0 [v2:172.21.15.152:6802/718734380,v1:172.21.15.152:6803/718734380]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-02T04:52:21.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:21 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:21.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:21 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:21.506 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 02 04:52:21 smithi152 ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-osd-0[48134]: 2024-04-02T04:52:21.106+0000 7f9d99168700 -1 osd.0 0 waiting for initial osdmap 2024-04-02T04:52:21.506 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 02 04:52:21 smithi152 ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-osd-0[48134]: 2024-04-02T04:52:21.108+0000 7f9d94f65700 -1 osd.0 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T04:52:21.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:21 smithi152 ceph-mon[29073]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:52:21.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:21 smithi152 ceph-mon[29073]: from='osd.0 [v2:172.21.15.152:6802/718734380,v1:172.21.15.152:6803/718734380]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-02T04:52:21.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:21 smithi152 ceph-mon[29073]: osdmap e7: 1 total, 0 up, 1 in 2024-04-02T04:52:21.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:21 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:52:21.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:21 smithi152 ceph-mon[29073]: from='osd.0 [v2:172.21.15.152:6802/718734380,v1:172.21.15.152:6803/718734380]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-02T04:52:21.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:21 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:21.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:21 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:21.578 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:52:22.356 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:22 smithi152 ceph-mon[29073]: from='osd.0 [v2:172.21.15.152:6802/718734380,v1:172.21.15.152:6803/718734380]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-02T04:52:22.356 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:22 smithi152 ceph-mon[29073]: osdmap e8: 1 total, 0 up, 1 in 2024-04-02T04:52:22.356 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:22 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:52:22.356 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:22 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:52:22.356 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:22 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:22.356 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:22 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:22.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:22 smithi154 ceph-mon[35729]: from='osd.0 [v2:172.21.15.152:6802/718734380,v1:172.21.15.152:6803/718734380]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-02T04:52:22.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:22 smithi154 ceph-mon[35729]: osdmap e8: 1 total, 0 up, 1 in 2024-04-02T04:52:22.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:22 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:52:22.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:22 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:52:22.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:22 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:22.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:22 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:23.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:23 smithi152 ceph-mon[29073]: purged_snaps scrub starts 2024-04-02T04:52:23.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:23 smithi152 ceph-mon[29073]: purged_snaps scrub ok 2024-04-02T04:52:23.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:23 smithi152 ceph-mon[29073]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:52:23.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:23 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:52:23.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:23 smithi152 ceph-mon[29073]: osd.0 [v2:172.21.15.152:6802/718734380,v1:172.21.15.152:6803/718734380] boot 2024-04-02T04:52:23.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:23 smithi152 ceph-mon[29073]: osdmap e9: 1 total, 1 up, 1 in 2024-04-02T04:52:23.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:23 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:52:23.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:23 smithi154 ceph-mon[35729]: purged_snaps scrub starts 2024-04-02T04:52:23.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:23 smithi154 ceph-mon[35729]: purged_snaps scrub ok 2024-04-02T04:52:23.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:23 smithi154 ceph-mon[35729]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-02T04:52:23.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:23 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:52:23.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:23 smithi154 ceph-mon[35729]: osd.0 [v2:172.21.15.152:6802/718734380,v1:172.21.15.152:6803/718734380] boot 2024-04-02T04:52:23.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:23 smithi154 ceph-mon[35729]: osdmap e9: 1 total, 1 up, 1 in 2024-04-02T04:52:23.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:23 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-02T04:52:24.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:24 smithi154 ceph-mon[35729]: osdmap e10: 1 total, 1 up, 1 in 2024-04-02T04:52:24.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:24 smithi152 ceph-mon[29073]: osdmap e10: 1 total, 1 up, 1 in 2024-04-02T04:52:24.874 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-02T04:52:24.874 INFO:teuthology.orchestra.run.smithi152.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-02T04:52:24.874 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10+0 records in 2024-04-02T04:52:24.874 INFO:teuthology.orchestra.run.smithi152.stderr:10+0 records out 2024-04-02T04:52:24.875 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0116058 s, 903 MB/s 2024-04-02T04:52:24.875 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping successful for: 2024-04-02T04:52:25.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:25 smithi154 ceph-mon[35729]: pgmap v35: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:25.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:25 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:52:25.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:25 smithi152 ceph-mon[29073]: pgmap v35: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:25.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:25 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:52:25.784 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi152:vg_nvme/lv_3 2024-04-02T04:52:26.590 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:52:27.556 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:27 smithi152 ceph-mon[29073]: pgmap v36: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:27.556 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:27 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:27.556 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:27 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:27.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:27 smithi154 ceph-mon[35729]: pgmap v36: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:27.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:27 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:27.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:27 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:28.061 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.057+0000 7fcd5993d700 1 -- 172.21.15.152:0/646345379 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcd540fcef0 msgr2=0x7fcd540fd300 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:52:28.062 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.057+0000 7fcd5993d700 1 --2- 172.21.15.152:0/646345379 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcd540fcef0 0x7fcd540fd300 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 crypto rx=0x7fcd44004940 tx=0x7fcd44009e30 comp rx=0 tx=0).stop 2024-04-02T04:52:28.062 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.057+0000 7fcd5993d700 1 -- 172.21.15.152:0/646345379 shutdown_connections 2024-04-02T04:52:28.062 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.057+0000 7fcd5993d700 1 --2- 172.21.15.152:0/646345379 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcd540fe060 0x7fcd540fe4d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:52:28.062 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.057+0000 7fcd5993d700 1 --2- 172.21.15.152:0/646345379 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcd540fcef0 0x7fcd540fd300 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:52:28.062 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.057+0000 7fcd5993d700 1 -- 172.21.15.152:0/646345379 >> 172.21.15.152:0/646345379 conn(0x7fcd540f85b0 msgr2=0x7fcd540fa9e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:52:28.062 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.057+0000 7fcd5993d700 1 -- 172.21.15.152:0/646345379 shutdown_connections 2024-04-02T04:52:28.062 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.057+0000 7fcd5993d700 1 -- 172.21.15.152:0/646345379 wait complete. 2024-04-02T04:52:28.064 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.058+0000 7fcd5993d700 1 Processor -- start 2024-04-02T04:52:28.064 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.058+0000 7fcd5993d700 1 -- start start 2024-04-02T04:52:28.064 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.058+0000 7fcd5993d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcd540fcef0 0x7fcd5410da80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:52:28.064 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.058+0000 7fcd5993d700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcd540fe060 0x7fcd5410dfc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:52:28.064 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.058+0000 7fcd5993d700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcd5410e5c0 con 0x7fcd540fcef0 2024-04-02T04:52:28.064 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.058+0000 7fcd5993d700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcd5410edc0 con 0x7fcd540fe060 2024-04-02T04:52:28.064 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.058+0000 7fcd52ffd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcd540fcef0 0x7fcd5410da80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:52:28.064 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.058+0000 7fcd52ffd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcd540fcef0 0x7fcd5410da80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:54110/0 (socket says 172.21.15.152:54110) 2024-04-02T04:52:28.064 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.058+0000 7fcd52ffd700 1 -- 172.21.15.152:0/1128925025 learned_addr learned my addr 172.21.15.152:0/1128925025 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:52:28.064 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.058+0000 7fcd527fc700 1 --2- 172.21.15.152:0/1128925025 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcd540fe060 0x7fcd5410dfc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:52:28.065 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.058+0000 7fcd52ffd700 1 -- 172.21.15.152:0/1128925025 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcd540fe060 msgr2=0x7fcd5410dfc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:52:28.065 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.058+0000 7fcd52ffd700 1 --2- 172.21.15.152:0/1128925025 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcd540fe060 0x7fcd5410dfc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:52:28.065 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.058+0000 7fcd52ffd700 1 -- 172.21.15.152:0/1128925025 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcd44005040 con 0x7fcd540fcef0 2024-04-02T04:52:28.065 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.058+0000 7fcd527fc700 1 --2- 172.21.15.152:0/1128925025 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcd540fe060 0x7fcd5410dfc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-02T04:52:28.065 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.058+0000 7fcd52ffd700 1 --2- 172.21.15.152:0/1128925025 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcd540fcef0 0x7fcd5410da80 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7fcd4400b3b0 tx=0x7fcd44009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:52:28.065 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.059+0000 7fcd3b7fe700 1 -- 172.21.15.152:0/1128925025 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fcd4400e040 con 0x7fcd540fcef0 2024-04-02T04:52:28.065 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.059+0000 7fcd3b7fe700 1 -- 172.21.15.152:0/1128925025 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fcd4400bcc0 con 0x7fcd540fcef0 2024-04-02T04:52:28.065 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.059+0000 7fcd5993d700 1 -- 172.21.15.152:0/1128925025 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcd5410f040 con 0x7fcd540fcef0 2024-04-02T04:52:28.065 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.059+0000 7fcd3b7fe700 1 -- 172.21.15.152:0/1128925025 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fcd4401c630 con 0x7fcd540fcef0 2024-04-02T04:52:28.065 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.059+0000 7fcd5993d700 1 -- 172.21.15.152:0/1128925025 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcd5410f580 con 0x7fcd540fcef0 2024-04-02T04:52:28.065 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.060+0000 7fcd5993d700 1 -- 172.21.15.152:0/1128925025 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcd5405f5f0 con 0x7fcd540fcef0 2024-04-02T04:52:28.066 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.060+0000 7fcd3b7fe700 1 -- 172.21.15.152:0/1128925025 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 18) v1 ==== 86567+0+0 (secure 0 0 0) 0x7fcd44013980 con 0x7fcd540fcef0 2024-04-02T04:52:28.066 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.060+0000 7fcd3b7fe700 1 --2- 172.21.15.152:0/1128925025 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fcd3c069210 0x7fcd3c06b6c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:52:28.066 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.061+0000 7fcd527fc700 1 --2- 172.21.15.152:0/1128925025 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fcd3c069210 0x7fcd3c06b6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:52:28.066 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.061+0000 7fcd3b7fe700 1 -- 172.21.15.152:0/1128925025 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(10..10 src has 1..10) v4 ==== 1978+0+0 (secure 0 0 0) 0x7fcd44087100 con 0x7fcd540fcef0 2024-04-02T04:52:28.066 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.061+0000 7fcd527fc700 1 --2- 172.21.15.152:0/1128925025 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fcd3c069210 0x7fcd3c06b6c0 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7fcd48009fd0 tx=0x7fcd48009ee0 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:52:28.070 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.065+0000 7fcd3b7fe700 1 -- 172.21.15.152:0/1128925025 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fcd44057cc0 con 0x7fcd540fcef0 2024-04-02T04:52:28.279 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:28.276+0000 7fcd5993d700 1 -- 172.21.15.152:0/1128925025 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_3", "target": ["mon-mgr", ""]}) v1 -- 0x7fcd5410f900 con 0x7fcd3c069210 2024-04-02T04:52:28.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:28 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:52:28.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:28 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:52:28.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:28 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:52:28.736 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:28 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:52:28.736 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:28 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:52:28.736 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:28 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:52:29.475 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:29 smithi152 ceph-mon[29073]: pgmap v37: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:29.475 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:29 smithi152 ceph-mon[29073]: from='client.14260 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:52:29.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:29 smithi154 ceph-mon[35729]: pgmap v37: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:29.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:29 smithi154 ceph-mon[35729]: from='client.14260 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:52:31.641 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:31 smithi152 ceph-mon[29073]: pgmap v38: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:31.641 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:31 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:31.641 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:31 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:31.641 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:31 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:52:31.641 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:31 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:31.641 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:31 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:52:31.641 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:31 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:52:31.641 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:31 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:31.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:31 smithi154 ceph-mon[35729]: pgmap v38: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:31.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:31 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:31.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:31 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:31.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:31 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:52:31.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:31 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:31.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:31 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:52:31.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:31 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:52:31.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:31 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:32.701 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:32 smithi154 ceph-mon[35729]: Detected new or changed devices on smithi152 2024-04-02T04:52:32.701 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:32 smithi154 ceph-mon[35729]: Adjusting osd_memory_target on smithi152 to 12864M 2024-04-02T04:52:32.701 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:52:32.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:32 smithi152 ceph-mon[29073]: Detected new or changed devices on smithi152 2024-04-02T04:52:32.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:32 smithi152 ceph-mon[29073]: Adjusting osd_memory_target on smithi152 to 12864M 2024-04-02T04:52:32.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:32 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:52:33.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:33 smithi154 ceph-mon[35729]: pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:33.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:33 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/4135534815' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d4555f9e-fc0e-4266-ae65-3c197d3fa02c"}]: dispatch 2024-04-02T04:52:33.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:33 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/4135534815' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d4555f9e-fc0e-4266-ae65-3c197d3fa02c"}]': finished 2024-04-02T04:52:33.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:33 smithi154 ceph-mon[35729]: osdmap e11: 2 total, 1 up, 2 in 2024-04-02T04:52:33.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:33 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:52:33.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:33 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:33.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:33 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:33.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:33 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/574450052' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:52:33.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:33 smithi152 ceph-mon[29073]: pgmap v39: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:33.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:33 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/4135534815' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d4555f9e-fc0e-4266-ae65-3c197d3fa02c"}]: dispatch 2024-04-02T04:52:33.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:33 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/4135534815' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d4555f9e-fc0e-4266-ae65-3c197d3fa02c"}]': finished 2024-04-02T04:52:33.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:33 smithi152 ceph-mon[29073]: osdmap e11: 2 total, 1 up, 2 in 2024-04-02T04:52:33.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:33 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:52:33.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:33 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:33.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:33 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:33.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:33 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/574450052' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:52:34.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:34 smithi154 ceph-mon[35729]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:35.006 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:34 smithi152 ceph-mon[29073]: pgmap v41: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:36.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:36 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:36.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:36 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:36.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:36 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:52:36.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:36 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:52:36.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:36 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:36.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:36 smithi152 ceph-mon[29073]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:36.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:36 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:36.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:36 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:36.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:36 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:52:36.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:36 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:52:36.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:36 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:36.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:36 smithi154 ceph-mon[35729]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:39.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:39 smithi152 ceph-mon[29073]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:39.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:39 smithi154 ceph-mon[35729]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:40.375 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:40 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-02T04:52:40.375 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:40 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:52:40.375 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:40 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:52:40.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:40 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-02T04:52:40.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:40 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:52:40.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:40 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:52:41.208 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:41 smithi152 ceph-mon[29073]: Deploying daemon osd.1 on smithi152 2024-04-02T04:52:41.208 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:41 smithi152 ceph-mon[29073]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:41.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:41 smithi154 ceph-mon[35729]: Deploying daemon osd.1 on smithi152 2024-04-02T04:52:41.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:41 smithi154 ceph-mon[35729]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:43.316 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:43 smithi152 ceph-mon[29073]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:43.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:43 smithi154 ceph-mon[35729]: pgmap v45: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:45.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:45 smithi152 ceph-mon[29073]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:45.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:45 smithi154 ceph-mon[35729]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:46.456 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:46 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:46.456 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:46 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:52:46.456 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:46 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:46.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:46 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:46.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:46 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:52:46.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:46 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:47.438 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:47 smithi152 ceph-mon[29073]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:47.486 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:47 smithi154 ceph-mon[35729]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:48.584 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:48.580+0000 7fcd3b7fe700 1 -- 172.21.15.152:0/1128925025 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7fcd5410f900 con 0x7fcd3c069210 2024-04-02T04:52:48.584 INFO:teuthology.orchestra.run.smithi152.stdout:Created osd(s) 1 on host 'smithi152' 2024-04-02T04:52:48.586 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:48.582+0000 7fcd5993d700 1 -- 172.21.15.152:0/1128925025 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fcd3c069210 msgr2=0x7fcd3c06b6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:52:48.586 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:48.582+0000 7fcd5993d700 1 --2- 172.21.15.152:0/1128925025 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fcd3c069210 0x7fcd3c06b6c0 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 crypto rx=0x7fcd48009fd0 tx=0x7fcd48009ee0 comp rx=0 tx=0).stop 2024-04-02T04:52:48.586 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:48.582+0000 7fcd5993d700 1 -- 172.21.15.152:0/1128925025 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcd540fcef0 msgr2=0x7fcd5410da80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:52:48.586 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:48.582+0000 7fcd5993d700 1 --2- 172.21.15.152:0/1128925025 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcd540fcef0 0x7fcd5410da80 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 crypto rx=0x7fcd4400b3b0 tx=0x7fcd44009e30 comp rx=0 tx=0).stop 2024-04-02T04:52:48.586 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:48.582+0000 7fcd5993d700 1 -- 172.21.15.152:0/1128925025 shutdown_connections 2024-04-02T04:52:48.587 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:48.582+0000 7fcd5993d700 1 --2- 172.21.15.152:0/1128925025 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fcd3c069210 0x7fcd3c06b6c0 unknown :-1 s=CLOSED pgs=31 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:52:48.587 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:48.582+0000 7fcd5993d700 1 --2- 172.21.15.152:0/1128925025 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcd540fcef0 0x7fcd5410da80 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:52:48.587 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:48.582+0000 7fcd5993d700 1 --2- 172.21.15.152:0/1128925025 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcd540fe060 0x7fcd5410dfc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:52:48.587 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:48.582+0000 7fcd5993d700 1 -- 172.21.15.152:0/1128925025 >> 172.21.15.152:0/1128925025 conn(0x7fcd540f85b0 msgr2=0x7fcd540f9cb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:52:48.587 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:48.583+0000 7fcd5993d700 1 -- 172.21.15.152:0/1128925025 shutdown_connections 2024-04-02T04:52:48.587 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:48.583+0000 7fcd5993d700 1 -- 172.21.15.152:0/1128925025 wait complete. 2024-04-02T04:52:49.389 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:49 smithi152 ceph-mon[29073]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:49.390 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:49 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:49.390 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:49 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:49.390 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:49 smithi152 ceph-mon[29073]: from='osd.1 [v2:172.21.15.152:6810/782616861,v1:172.21.15.152:6811/782616861]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-02T04:52:49.418 DEBUG:teuthology.orchestra.run.smithi152:osd.1> sudo journalctl -f -n 0 -u ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4@osd.1.service 2024-04-02T04:52:49.420 INFO:tasks.cephadm:Deploying osd.2 on smithi152 with /dev/vg_nvme/lv_2... 2024-04-02T04:52:49.420 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-02T04:52:49.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:49 smithi154 ceph-mon[35729]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:49.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:49 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:49.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:49 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:49.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:49 smithi154 ceph-mon[35729]: from='osd.1 [v2:172.21.15.152:6810/782616861,v1:172.21.15.152:6811/782616861]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-02T04:52:50.135 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:52:50.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:50 smithi154 ceph-mon[35729]: from='osd.1 [v2:172.21.15.152:6810/782616861,v1:172.21.15.152:6811/782616861]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-02T04:52:50.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:50 smithi154 ceph-mon[35729]: osdmap e12: 2 total, 1 up, 2 in 2024-04-02T04:52:50.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:50 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:52:50.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:50 smithi154 ceph-mon[35729]: from='osd.1 [v2:172.21.15.152:6810/782616861,v1:172.21.15.152:6811/782616861]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-02T04:52:50.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:50 smithi154 ceph-mon[35729]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:51.006 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:50 smithi152 ceph-mon[29073]: from='osd.1 [v2:172.21.15.152:6810/782616861,v1:172.21.15.152:6811/782616861]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-02T04:52:51.006 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:50 smithi152 ceph-mon[29073]: osdmap e12: 2 total, 1 up, 2 in 2024-04-02T04:52:51.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:50 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:52:51.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:50 smithi152 ceph-mon[29073]: from='osd.1 [v2:172.21.15.152:6810/782616861,v1:172.21.15.152:6811/782616861]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-02T04:52:51.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:50 smithi152 ceph-mon[29073]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-04-02T04:52:51.007 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 02 04:52:50 smithi152 ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-osd-1[56254]: 2024-04-02T04:52:50.573+0000 7fded922e700 -1 osd.1 0 waiting for initial osdmap 2024-04-02T04:52:51.007 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 02 04:52:50 smithi152 ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-osd-1[56254]: 2024-04-02T04:52:50.577+0000 7fded2826700 -1 osd.1 13 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T04:52:51.893 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:51 smithi152 ceph-mon[29073]: from='osd.1 [v2:172.21.15.152:6810/782616861,v1:172.21.15.152:6811/782616861]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-02T04:52:51.893 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:51 smithi152 ceph-mon[29073]: osdmap e13: 2 total, 1 up, 2 in 2024-04-02T04:52:51.893 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:51 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:52:51.893 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:51 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:52:51.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:51 smithi154 ceph-mon[35729]: from='osd.1 [v2:172.21.15.152:6810/782616861,v1:172.21.15.152:6811/782616861]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-02T04:52:51.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:51 smithi154 ceph-mon[35729]: osdmap e13: 2 total, 1 up, 2 in 2024-04-02T04:52:51.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:51 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:52:51.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:51 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:52:52.634 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:52 smithi152 ceph-mon[29073]: purged_snaps scrub starts 2024-04-02T04:52:52.634 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:52 smithi152 ceph-mon[29073]: purged_snaps scrub ok 2024-04-02T04:52:52.634 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:52 smithi152 ceph-mon[29073]: osd.1 [v2:172.21.15.152:6810/782616861,v1:172.21.15.152:6811/782616861] boot 2024-04-02T04:52:52.634 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:52 smithi152 ceph-mon[29073]: osdmap e14: 2 total, 2 up, 2 in 2024-04-02T04:52:52.634 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:52 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:52:52.634 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:52 smithi152 ceph-mon[29073]: pgmap v53: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:52:52.821 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-02T04:52:52.821 INFO:teuthology.orchestra.run.smithi152.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-02T04:52:52.822 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10+0 records in 2024-04-02T04:52:52.822 INFO:teuthology.orchestra.run.smithi152.stderr:10+0 records out 2024-04-02T04:52:52.822 INFO:teuthology.orchestra.run.smithi152.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0173456 s, 605 MB/s 2024-04-02T04:52:52.822 INFO:teuthology.orchestra.run.smithi152.stderr:--> Zapping successful for: 2024-04-02T04:52:52.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:52 smithi154 ceph-mon[35729]: purged_snaps scrub starts 2024-04-02T04:52:52.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:52 smithi154 ceph-mon[35729]: purged_snaps scrub ok 2024-04-02T04:52:52.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:52 smithi154 ceph-mon[35729]: osd.1 [v2:172.21.15.152:6810/782616861,v1:172.21.15.152:6811/782616861] boot 2024-04-02T04:52:52.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:52 smithi154 ceph-mon[35729]: osdmap e14: 2 total, 2 up, 2 in 2024-04-02T04:52:52.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:52 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-02T04:52:52.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:52 smithi154 ceph-mon[35729]: pgmap v53: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:52:53.612 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi152:vg_nvme/lv_2 2024-04-02T04:52:53.902 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:52:53.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:53 smithi154 ceph-mon[35729]: osdmap e15: 2 total, 2 up, 2 in 2024-04-02T04:52:54.006 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:53 smithi152 ceph-mon[29073]: osdmap e15: 2 total, 2 up, 2 in 2024-04-02T04:52:54.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:54 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:54.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:54 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:54.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:54 smithi154 ceph-mon[35729]: pgmap v55: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:52:54.987 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:54 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:54.987 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:54 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:54.987 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:54 smithi152 ceph-mon[29073]: pgmap v55: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:52:55.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:55 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:52:56.006 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:55 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:52:56.212 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.208+0000 7fd3cc033700 1 -- 172.21.15.152:0/128005121 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd3c40fff10 msgr2=0x7fd3c4102330 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:52:56.212 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.208+0000 7fd3cc033700 1 --2- 172.21.15.152:0/128005121 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd3c40fff10 0x7fd3c4102330 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 crypto rx=0x7fd3b4004940 tx=0x7fd3b4009e30 comp rx=0 tx=0).stop 2024-04-02T04:52:56.212 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.209+0000 7fd3cc033700 1 -- 172.21.15.152:0/128005121 shutdown_connections 2024-04-02T04:52:56.213 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.209+0000 7fd3cc033700 1 --2- 172.21.15.152:0/128005121 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd3c40fff10 0x7fd3c4102330 unknown :-1 s=CLOSED pgs=164 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:52:56.213 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.209+0000 7fd3cc033700 1 --2- 172.21.15.152:0/128005121 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd3c4066fd0 0x7fd3c40ff9d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:52:56.213 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.209+0000 7fd3cc033700 1 -- 172.21.15.152:0/128005121 >> 172.21.15.152:0/128005121 conn(0x7fd3c40f7440 msgr2=0x7fd3c40f9870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:52:56.213 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.209+0000 7fd3cc033700 1 -- 172.21.15.152:0/128005121 shutdown_connections 2024-04-02T04:52:56.213 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.209+0000 7fd3cc033700 1 -- 172.21.15.152:0/128005121 wait complete. 2024-04-02T04:52:56.215 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.209+0000 7fd3cc033700 1 Processor -- start 2024-04-02T04:52:56.215 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.209+0000 7fd3cc033700 1 -- start start 2024-04-02T04:52:56.215 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.210+0000 7fd3cc033700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd3c4066fd0 0x7fd3c410d980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:52:56.215 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.210+0000 7fd3cc033700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd3c40fff10 0x7fd3c410dec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:52:56.215 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.210+0000 7fd3cc033700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd3c410e4c0 con 0x7fd3c40fff10 2024-04-02T04:52:56.215 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.210+0000 7fd3cc033700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd3c410e630 con 0x7fd3c4066fd0 2024-04-02T04:52:56.215 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.210+0000 7fd3c95ce700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd3c40fff10 0x7fd3c410dec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:52:56.215 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.210+0000 7fd3c95ce700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd3c40fff10 0x7fd3c410dec0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:52720/0 (socket says 172.21.15.152:52720) 2024-04-02T04:52:56.215 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.210+0000 7fd3c95ce700 1 -- 172.21.15.152:0/3787745320 learned_addr learned my addr 172.21.15.152:0/3787745320 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:52:56.215 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.210+0000 7fd3c9dcf700 1 --2- 172.21.15.152:0/3787745320 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd3c4066fd0 0x7fd3c410d980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:52:56.215 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.210+0000 7fd3c95ce700 1 -- 172.21.15.152:0/3787745320 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd3c4066fd0 msgr2=0x7fd3c410d980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:52:56.216 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.210+0000 7fd3c95ce700 1 --2- 172.21.15.152:0/3787745320 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd3c4066fd0 0x7fd3c410d980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:52:56.216 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.210+0000 7fd3c95ce700 1 -- 172.21.15.152:0/3787745320 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd3b4005040 con 0x7fd3c40fff10 2024-04-02T04:52:56.216 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.210+0000 7fd3c95ce700 1 --2- 172.21.15.152:0/3787745320 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd3c40fff10 0x7fd3c410dec0 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7fd3b4009150 tx=0x7fd3b4009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:52:56.216 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.210+0000 7fd3ba7fc700 1 -- 172.21.15.152:0/3787745320 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fd3b400e040 con 0x7fd3c40fff10 2024-04-02T04:52:56.216 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.210+0000 7fd3cc033700 1 -- 172.21.15.152:0/3787745320 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd3c4113030 con 0x7fd3c40fff10 2024-04-02T04:52:56.216 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.210+0000 7fd3ba7fc700 1 -- 172.21.15.152:0/3787745320 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fd3b4004290 con 0x7fd3c40fff10 2024-04-02T04:52:56.216 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.210+0000 7fd3ba7fc700 1 -- 172.21.15.152:0/3787745320 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fd3b401c630 con 0x7fd3c40fff10 2024-04-02T04:52:56.216 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.210+0000 7fd3cc033700 1 -- 172.21.15.152:0/3787745320 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd3c41135f0 con 0x7fd3c40fff10 2024-04-02T04:52:56.216 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.211+0000 7fd3cc033700 1 -- 172.21.15.152:0/3787745320 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd3c404c710 con 0x7fd3c40fff10 2024-04-02T04:52:56.217 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.212+0000 7fd3ba7fc700 1 -- 172.21.15.152:0/3787745320 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 18) v1 ==== 86567+0+0 (secure 0 0 0) 0x7fd3b400bbf0 con 0x7fd3c40fff10 2024-04-02T04:52:56.217 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.212+0000 7fd3ba7fc700 1 --2- 172.21.15.152:0/3787745320 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd3b0069210 0x7fd3b006b6c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:52:56.217 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.212+0000 7fd3ba7fc700 1 -- 172.21.15.152:0/3787745320 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(15..15 src has 1..15) v4 ==== 2410+0+0 (secure 0 0 0) 0x7fd3b4087060 con 0x7fd3c40fff10 2024-04-02T04:52:56.217 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.212+0000 7fd3c9dcf700 1 --2- 172.21.15.152:0/3787745320 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd3b0069210 0x7fd3b006b6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:52:56.217 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.212+0000 7fd3c9dcf700 1 --2- 172.21.15.152:0/3787745320 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd3b0069210 0x7fd3b006b6c0 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7fd3c4066d10 tx=0x7fd3c000b040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:52:56.220 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.217+0000 7fd3ba7fc700 1 -- 172.21.15.152:0/3787745320 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fd3b4057b90 con 0x7fd3c40fff10 2024-04-02T04:52:56.417 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:52:56.413+0000 7fd3cc033700 1 -- 172.21.15.152:0/3787745320 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_2", "target": ["mon-mgr", ""]}) v1 -- 0x7fd3c40f8f80 con 0x7fd3b0069210 2024-04-02T04:52:56.939 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:56 smithi152 ceph-mon[29073]: pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:52:56.940 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:56 smithi152 ceph-mon[29073]: from='client.14278 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:52:56.940 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:56 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:52:56.940 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:56 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:52:56.940 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:56 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:52:56.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:56 smithi154 ceph-mon[35729]: pgmap v56: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:52:56.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:56 smithi154 ceph-mon[35729]: from='client.14278 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi152:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:52:56.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:56 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:52:56.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:56 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:52:56.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:56 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:52:59.241 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:59 smithi152 ceph-mon[29073]: pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:52:59.241 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:59 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:59.241 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:59 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:59.241 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:59 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:52:59.241 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:59 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:52:59.241 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:59 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:59.241 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:59 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:52:59.241 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:59 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:52:59.241 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:52:59 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:59.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:59 smithi154 ceph-mon[35729]: pgmap v57: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:52:59.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:59 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:59.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:59 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:59.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:59 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:52:59.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:59 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:52:59.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:59 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:52:59.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:59 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:52:59.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:59 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:52:59.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:52:59 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:00.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:00 smithi152 ceph-mon[29073]: Detected new or changed devices on smithi152 2024-04-02T04:53:00.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:00 smithi152 ceph-mon[29073]: Adjusting osd_memory_target on smithi152 to 6432M 2024-04-02T04:53:00.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:00 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:53:00.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:00 smithi154 ceph-mon[35729]: Detected new or changed devices on smithi152 2024-04-02T04:53:00.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:00 smithi154 ceph-mon[35729]: Adjusting osd_memory_target on smithi152 to 6432M 2024-04-02T04:53:00.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:00 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:53:01.300 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:01 smithi152 ceph-mon[29073]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:53:01.300 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:01 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/2710400611' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "caa0a912-a9d5-45e7-afe2-51beaf0f1742"}]: dispatch 2024-04-02T04:53:01.300 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:01 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/2710400611' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "caa0a912-a9d5-45e7-afe2-51beaf0f1742"}]': finished 2024-04-02T04:53:01.300 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:01 smithi152 ceph-mon[29073]: osdmap e16: 3 total, 2 up, 3 in 2024-04-02T04:53:01.300 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:01 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:53:01.300 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:01 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/173708606' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:53:01.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:01 smithi154 ceph-mon[35729]: pgmap v58: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:53:01.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:01 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/2710400611' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "caa0a912-a9d5-45e7-afe2-51beaf0f1742"}]: dispatch 2024-04-02T04:53:01.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:01 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/2710400611' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "caa0a912-a9d5-45e7-afe2-51beaf0f1742"}]': finished 2024-04-02T04:53:01.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:01 smithi154 ceph-mon[35729]: osdmap e16: 3 total, 2 up, 3 in 2024-04-02T04:53:01.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:01 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:53:01.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:01 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/173708606' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:53:03.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:03 smithi154 ceph-mon[35729]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:53:03.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:03 smithi152 ceph-mon[29073]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:53:04.359 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:04 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:04.360 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:04 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:04.360 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:04 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:53:04.360 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:04 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:53:04.360 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:04 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:04.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:04 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:04.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:04 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:04.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:04 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:53:04.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:04 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:53:04.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:04 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:05.439 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:05 smithi152 ceph-mon[29073]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:53:05.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:05 smithi154 ceph-mon[35729]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:53:07.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:07 smithi154 ceph-mon[35729]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:53:07.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:07 smithi152 ceph-mon[29073]: pgmap v62: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:53:08.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:08 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-02T04:53:08.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:08 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:53:08.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:08 smithi152 ceph-mon[29073]: Deploying daemon osd.2 on smithi152 2024-04-02T04:53:08.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:08 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-02T04:53:08.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:08 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:53:08.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:08 smithi154 ceph-mon[35729]: Deploying daemon osd.2 on smithi152 2024-04-02T04:53:09.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:09 smithi152 ceph-mon[29073]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:53:09.634 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:09 smithi154 ceph-mon[35729]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:53:10.371 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:10 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:53:10.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:10 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:53:11.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:11 smithi152 ceph-mon[29073]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:53:11.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:11 smithi154 ceph-mon[35729]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:53:13.451 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:13 smithi152 ceph-mon[29073]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:53:13.451 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:13 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:13.451 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:13 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:13.451 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:13 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:53:13.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:13 smithi154 ceph-mon[35729]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:53:13.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:13 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:13.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:13 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:13.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:13 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:53:15.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:15 smithi152 ceph-mon[29073]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:53:15.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:15 smithi154 ceph-mon[35729]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:53:15.871 INFO:teuthology.orchestra.run.smithi152.stdout:Created osd(s) 2 on host 'smithi152' 2024-04-02T04:53:15.871 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:53:15.867+0000 7fd3ba7fc700 1 -- 172.21.15.152:0/3787745320 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7fd3c40f8f80 con 0x7fd3b0069210 2024-04-02T04:53:15.873 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:53:15.869+0000 7fd3cc033700 1 -- 172.21.15.152:0/3787745320 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd3b0069210 msgr2=0x7fd3b006b6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:53:15.873 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:53:15.869+0000 7fd3cc033700 1 --2- 172.21.15.152:0/3787745320 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd3b0069210 0x7fd3b006b6c0 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 crypto rx=0x7fd3c4066d10 tx=0x7fd3c000b040 comp rx=0 tx=0).stop 2024-04-02T04:53:15.873 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:53:15.869+0000 7fd3cc033700 1 -- 172.21.15.152:0/3787745320 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd3c40fff10 msgr2=0x7fd3c410dec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:53:15.873 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:53:15.869+0000 7fd3cc033700 1 --2- 172.21.15.152:0/3787745320 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd3c40fff10 0x7fd3c410dec0 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 crypto rx=0x7fd3b4009150 tx=0x7fd3b4009e30 comp rx=0 tx=0).stop 2024-04-02T04:53:15.873 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:53:15.869+0000 7fd3cc033700 1 -- 172.21.15.152:0/3787745320 shutdown_connections 2024-04-02T04:53:15.873 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:53:15.869+0000 7fd3cc033700 1 --2- 172.21.15.152:0/3787745320 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd3b0069210 0x7fd3b006b6c0 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:53:15.873 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:53:15.869+0000 7fd3cc033700 1 --2- 172.21.15.152:0/3787745320 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd3c4066fd0 0x7fd3c410d980 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:53:15.874 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:53:15.869+0000 7fd3cc033700 1 --2- 172.21.15.152:0/3787745320 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd3c40fff10 0x7fd3c410dec0 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:53:15.874 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:53:15.869+0000 7fd3cc033700 1 -- 172.21.15.152:0/3787745320 >> 172.21.15.152:0/3787745320 conn(0x7fd3c40f7440 msgr2=0x7fd3c40f7860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:53:15.874 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:53:15.869+0000 7fd3cc033700 1 -- 172.21.15.152:0/3787745320 shutdown_connections 2024-04-02T04:53:15.874 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:53:15.869+0000 7fd3cc033700 1 -- 172.21.15.152:0/3787745320 wait complete. 2024-04-02T04:53:16.493 DEBUG:teuthology.orchestra.run.smithi152:osd.2> sudo journalctl -f -n 0 -u ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4@osd.2.service 2024-04-02T04:53:16.495 INFO:tasks.cephadm:Deploying osd.3 on smithi154 with /dev/vg_nvme/lv_4... 2024-04-02T04:53:16.496 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-02T04:53:16.757 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 02 04:53:16 smithi152 ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-osd-2[64055]: 2024-04-02T04:53:16.670+0000 7f78407a5540 -1 osd.2 0 log_to_monitors true 2024-04-02T04:53:16.757 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi154/config 2024-04-02T04:53:17.135 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:16 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:17.136 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:16 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:17.136 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:16 smithi154 ceph-mon[35729]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:53:17.136 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:16 smithi154 ceph-mon[35729]: from='osd.2 [v2:172.21.15.152:6818/1566571469,v1:172.21.15.152:6819/1566571469]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-02T04:53:17.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:16 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:17.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:16 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:17.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:16 smithi152 ceph-mon[29073]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:53:17.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:16 smithi152 ceph-mon[29073]: from='osd.2 [v2:172.21.15.152:6818/1566571469,v1:172.21.15.152:6819/1566571469]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-02T04:53:17.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:17 smithi154 ceph-mon[35729]: from='osd.2 [v2:172.21.15.152:6818/1566571469,v1:172.21.15.152:6819/1566571469]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-02T04:53:17.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:17 smithi154 ceph-mon[35729]: osdmap e17: 3 total, 2 up, 3 in 2024-04-02T04:53:17.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:17 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:53:17.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:17 smithi154 ceph-mon[35729]: from='osd.2 [v2:172.21.15.152:6818/1566571469,v1:172.21.15.152:6819/1566571469]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-02T04:53:18.134 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:17 smithi152 ceph-mon[29073]: from='osd.2 [v2:172.21.15.152:6818/1566571469,v1:172.21.15.152:6819/1566571469]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-02T04:53:18.134 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:17 smithi152 ceph-mon[29073]: osdmap e17: 3 total, 2 up, 3 in 2024-04-02T04:53:18.134 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:17 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:53:18.134 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:17 smithi152 ceph-mon[29073]: from='osd.2 [v2:172.21.15.152:6818/1566571469,v1:172.21.15.152:6819/1566571469]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]: dispatch 2024-04-02T04:53:18.135 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 02 04:53:17 smithi152 ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-osd-2[64055]: 2024-04-02T04:53:17.878+0000 7f7836f77700 -1 osd.2 0 waiting for initial osdmap 2024-04-02T04:53:18.135 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 02 04:53:17 smithi152 ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-osd-2[64055]: 2024-04-02T04:53:17.885+0000 7f7832573700 -1 osd.2 18 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T04:53:18.565 INFO:teuthology.orchestra.run.smithi154.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-02T04:53:18.565 INFO:teuthology.orchestra.run.smithi154.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-02T04:53:18.565 INFO:teuthology.orchestra.run.smithi154.stderr: stderr: 10+0 records in 2024-04-02T04:53:18.565 INFO:teuthology.orchestra.run.smithi154.stderr:10+0 records out 2024-04-02T04:53:18.565 INFO:teuthology.orchestra.run.smithi154.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0113496 s, 924 MB/s 2024-04-02T04:53:18.565 INFO:teuthology.orchestra.run.smithi154.stderr:--> Zapping successful for: 2024-04-02T04:53:19.145 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:18 smithi154 ceph-mon[35729]: from='osd.2 [v2:172.21.15.152:6818/1566571469,v1:172.21.15.152:6819/1566571469]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-02T04:53:19.145 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:18 smithi154 ceph-mon[35729]: osdmap e18: 3 total, 2 up, 3 in 2024-04-02T04:53:19.145 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:18 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:53:19.145 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:18 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:53:19.145 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:18 smithi154 ceph-mon[35729]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:53:19.146 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi154:vg_nvme/lv_4 2024-04-02T04:53:19.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:18 smithi152 ceph-mon[29073]: from='osd.2 [v2:172.21.15.152:6818/1566571469,v1:172.21.15.152:6819/1566571469]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi152", "root=default"]}]': finished 2024-04-02T04:53:19.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:18 smithi152 ceph-mon[29073]: osdmap e18: 3 total, 2 up, 3 in 2024-04-02T04:53:19.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:18 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:53:19.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:18 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:53:19.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:18 smithi152 ceph-mon[29073]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-04-02T04:53:19.410 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi154/config 2024-04-02T04:53:20.102 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:19 smithi154 ceph-mon[35729]: purged_snaps scrub starts 2024-04-02T04:53:20.102 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:19 smithi154 ceph-mon[35729]: purged_snaps scrub ok 2024-04-02T04:53:20.102 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:19 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:53:20.102 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:19 smithi154 ceph-mon[35729]: osd.2 [v2:172.21.15.152:6818/1566571469,v1:172.21.15.152:6819/1566571469] boot 2024-04-02T04:53:20.102 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:19 smithi154 ceph-mon[35729]: osdmap e19: 3 total, 3 up, 3 in 2024-04-02T04:53:20.103 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:19 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:53:20.103 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:19 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:20.103 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:19 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:20.107 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:19 smithi152 ceph-mon[29073]: purged_snaps scrub starts 2024-04-02T04:53:20.107 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:19 smithi152 ceph-mon[29073]: purged_snaps scrub ok 2024-04-02T04:53:20.107 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:19 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:53:20.107 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:19 smithi152 ceph-mon[29073]: osd.2 [v2:172.21.15.152:6818/1566571469,v1:172.21.15.152:6819/1566571469] boot 2024-04-02T04:53:20.107 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:19 smithi152 ceph-mon[29073]: osdmap e19: 3 total, 3 up, 3 in 2024-04-02T04:53:20.107 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:19 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-02T04:53:20.107 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:19 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:20.107 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:19 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:20.840 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.835+0000 7f3a93630700 1 -- 172.21.15.154:0/1813185866 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3a8c0fcee0 msgr2=0x7f3a8c0fd310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:53:20.840 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.835+0000 7f3a93630700 1 --2- 172.21.15.154:0/1813185866 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3a8c0fcee0 0x7f3a8c0fd310 secure :-1 s=READY pgs=4 cs=0 l=1 rev1=1 crypto rx=0x7f3a88004970 tx=0x7f3a88009e30 comp rx=0 tx=0).stop 2024-04-02T04:53:20.841 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.835+0000 7f3a93630700 1 -- 172.21.15.154:0/1813185866 shutdown_connections 2024-04-02T04:53:20.841 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.835+0000 7f3a93630700 1 --2- 172.21.15.154:0/1813185866 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3a8c0fe0d0 0x7f3a8c0fe540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:53:20.841 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.835+0000 7f3a93630700 1 --2- 172.21.15.154:0/1813185866 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3a8c0fcee0 0x7f3a8c0fd310 unknown :-1 s=CLOSED pgs=4 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:53:20.841 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.835+0000 7f3a93630700 1 -- 172.21.15.154:0/1813185866 >> 172.21.15.154:0/1813185866 conn(0x7f3a8c0f8560 msgr2=0x7f3a8c0fa970 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:53:20.841 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.835+0000 7f3a93630700 1 -- 172.21.15.154:0/1813185866 shutdown_connections 2024-04-02T04:53:20.841 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.835+0000 7f3a93630700 1 -- 172.21.15.154:0/1813185866 wait complete. 2024-04-02T04:53:20.841 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.836+0000 7f3a93630700 1 Processor -- start 2024-04-02T04:53:20.841 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.836+0000 7f3a93630700 1 -- start start 2024-04-02T04:53:20.841 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.836+0000 7f3a93630700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3a8c0fcee0 0x7f3a8c10da00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:53:20.841 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.836+0000 7f3a93630700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3a8c0fe0d0 0x7f3a8c10df40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:53:20.841 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.836+0000 7f3a93630700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3a8c10e540 con 0x7f3a8c0fcee0 2024-04-02T04:53:20.841 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.836+0000 7f3a93630700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3a8c10ed40 con 0x7f3a8c0fe0d0 2024-04-02T04:53:20.842 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.836+0000 7f3a90bcb700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3a8c0fe0d0 0x7f3a8c10df40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:53:20.842 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.836+0000 7f3a90bcb700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3a8c0fe0d0 0x7f3a8c10df40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.154:33746/0 (socket says 172.21.15.154:33746) 2024-04-02T04:53:20.842 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.836+0000 7f3a90bcb700 1 -- 172.21.15.154:0/611754367 learned_addr learned my addr 172.21.15.154:0/611754367 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-04-02T04:53:20.842 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.836+0000 7f3a913cc700 1 --2- 172.21.15.154:0/611754367 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3a8c0fcee0 0x7f3a8c10da00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:53:20.842 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.836+0000 7f3a90bcb700 1 -- 172.21.15.154:0/611754367 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3a8c0fcee0 msgr2=0x7f3a8c10da00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:53:20.842 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.836+0000 7f3a90bcb700 1 --2- 172.21.15.154:0/611754367 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3a8c0fcee0 0x7f3a8c10da00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:53:20.842 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.836+0000 7f3a90bcb700 1 -- 172.21.15.154:0/611754367 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3a88005040 con 0x7f3a8c0fe0d0 2024-04-02T04:53:20.842 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.837+0000 7f3a90bcb700 1 --2- 172.21.15.154:0/611754367 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3a8c0fe0d0 0x7f3a8c10df40 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7f3a80002a50 tx=0x7f3a800046b0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:53:20.842 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.837+0000 7f3a7dffb700 1 -- 172.21.15.154:0/611754367 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f3a80007cc0 con 0x7f3a8c0fe0d0 2024-04-02T04:53:20.842 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.837+0000 7f3a7dffb700 1 -- 172.21.15.154:0/611754367 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f3a8000f040 con 0x7f3a8c0fe0d0 2024-04-02T04:53:20.842 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.837+0000 7f3a93630700 1 -- 172.21.15.154:0/611754367 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3a8c10f020 con 0x7f3a8c0fe0d0 2024-04-02T04:53:20.842 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.837+0000 7f3a7dffb700 1 -- 172.21.15.154:0/611754367 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f3a8000da30 con 0x7f3a8c0fe0d0 2024-04-02T04:53:20.843 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.837+0000 7f3a93630700 1 -- 172.21.15.154:0/611754367 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3a8c10f560 con 0x7f3a8c0fe0d0 2024-04-02T04:53:20.843 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.838+0000 7f3a93630700 1 -- 172.21.15.154:0/611754367 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3a8c05f5f0 con 0x7f3a8c0fe0d0 2024-04-02T04:53:20.843 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.838+0000 7f3a7dffb700 1 -- 172.21.15.154:0/611754367 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 18) v1 ==== 86567+0+0 (secure 0 0 0) 0x7f3a8000db90 con 0x7f3a8c0fe0d0 2024-04-02T04:53:20.844 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.839+0000 7f3a7dffb700 1 --2- 172.21.15.154:0/611754367 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f3a78069210 0x7f3a7806b6c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:53:20.844 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.839+0000 7f3a7dffb700 1 -- 172.21.15.154:0/611754367 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(20..20 src has 1..20) v4 ==== 3217+0+0 (secure 0 0 0) 0x7f3a800864c0 con 0x7f3a8c0fe0d0 2024-04-02T04:53:20.844 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.839+0000 7f3a913cc700 1 --2- 172.21.15.154:0/611754367 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f3a78069210 0x7f3a7806b6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:53:20.844 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.839+0000 7f3a913cc700 1 --2- 172.21.15.154:0/611754367 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f3a78069210 0x7f3a7806b6c0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f3a88009fd0 tx=0x7f3a88009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:53:20.846 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:20.843+0000 7f3a7dffb700 1 -- 172.21.15.154:0/611754367 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f3a80056c20 con 0x7f3a8c0fe0d0 2024-04-02T04:53:20.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:20 smithi154 ceph-mon[35729]: pgmap v72: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-02T04:53:20.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:20 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-02T04:53:20.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:20 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-02T04:53:20.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:20 smithi154 ceph-mon[35729]: osdmap e20: 3 total, 3 up, 3 in 2024-04-02T04:53:20.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:20 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-02T04:53:21.040 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:21.036+0000 7f3a93630700 1 -- 172.21.15.154:0/611754367 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_4", "target": ["mon-mgr", ""]}) v1 -- 0x7f3a8c10f8e0 con 0x7f3a78069210 2024-04-02T04:53:21.157 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:20 smithi152 ceph-mon[29073]: pgmap v72: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-02T04:53:21.157 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:20 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-02T04:53:21.157 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:20 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-02T04:53:21.157 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:20 smithi152 ceph-mon[29073]: osdmap e20: 3 total, 3 up, 3 in 2024-04-02T04:53:21.157 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:20 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-02T04:53:21.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:21 smithi154 ceph-mon[35729]: from='client.24135 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:53:21.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:21 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:53:21.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:21 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:53:21.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:21 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:53:21.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:21 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-02T04:53:21.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:21 smithi154 ceph-mon[35729]: osdmap e21: 3 total, 3 up, 3 in 2024-04-02T04:53:22.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:21 smithi152 ceph-mon[29073]: from='client.24135 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:53:22.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:21 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:53:22.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:21 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:53:22.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:21 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:53:22.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:21 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-02T04:53:22.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:21 smithi152 ceph-mon[29073]: osdmap e21: 3 total, 3 up, 3 in 2024-04-02T04:53:22.893 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 02 04:53:22 smithi152 sudo[68422]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T04:53:22.894 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 02 04:53:22 smithi152 sudo[68422]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-02T04:53:22.894 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 02 04:53:22 smithi152 sudo[68422]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-02T04:53:22.894 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 02 04:53:22 smithi152 sudo[68422]: pam_unix(sudo:session): session closed for user root 2024-04-02T04:53:22.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:22 smithi154 ceph-mon[35729]: pgmap v75: 1 pgs: 1 creating+activating; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-02T04:53:22.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:22 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:22.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:22 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:22.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:22 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:53:22.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:22 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:53:22.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:22 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:53:22.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:22 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:22.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:22 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:53:22.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:22 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:53:22.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:22 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:22.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:22 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:53:23.229 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 02 04:53:23 smithi152 sudo[68476]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T04:53:23.229 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 02 04:53:23 smithi152 sudo[68476]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-02T04:53:23.230 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 02 04:53:23 smithi152 sudo[68476]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-02T04:53:23.230 INFO:journalctl@ceph.osd.0.smithi152.stdout:Apr 02 04:53:23 smithi152 sudo[68476]: pam_unix(sudo:session): session closed for user root 2024-04-02T04:53:23.230 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:22 smithi152 ceph-mon[29073]: pgmap v75: 1 pgs: 1 creating+activating; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-02T04:53:23.230 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:22 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:23.230 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:22 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:23.230 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:22 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:53:23.230 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:22 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:53:23.230 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:22 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:53:23.230 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:22 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:23.230 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:22 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:53:23.230 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:22 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:53:23.230 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:22 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:23.231 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:22 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:53:23.506 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 02 04:53:23 smithi152 sudo[68523]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T04:53:23.506 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 02 04:53:23 smithi152 sudo[68523]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-02T04:53:23.507 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 02 04:53:23 smithi152 sudo[68523]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-02T04:53:23.507 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 02 04:53:23 smithi152 sudo[68523]: pam_unix(sudo:session): session closed for user root 2024-04-02T04:53:23.850 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 02 04:53:23 smithi152 sudo[68526]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T04:53:23.850 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 02 04:53:23 smithi152 sudo[68526]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-02T04:53:23.850 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 02 04:53:23 smithi152 sudo[68526]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-02T04:53:23.850 INFO:journalctl@ceph.osd.1.smithi152.stdout:Apr 02 04:53:23 smithi152 sudo[68526]: pam_unix(sudo:session): session closed for user root 2024-04-02T04:53:23.850 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 02 04:53:23 smithi152 sudo[68529]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T04:53:23.850 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 02 04:53:23 smithi152 sudo[68529]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-02T04:53:23.850 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 02 04:53:23 smithi152 sudo[68529]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-02T04:53:23.850 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 02 04:53:23 smithi152 sudo[68529]: pam_unix(sudo:session): session closed for user root 2024-04-02T04:53:24.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:23 smithi154 ceph-mon[35729]: Detected new or changed devices on smithi152 2024-04-02T04:53:24.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:23 smithi154 ceph-mon[35729]: Adjusting osd_memory_target on smithi152 to 4288M 2024-04-02T04:53:24.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:23 smithi154 ceph-mon[35729]: osdmap e22: 3 total, 3 up, 3 in 2024-04-02T04:53:24.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:23 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:24.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:23 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:53:24.234 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:23 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:53:24.234 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:23 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:24.234 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:23 smithi154 ceph-mon[35729]: from='client.? 172.21.15.154:0/3731552346' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84a2d337-6ac8-41e4-a922-a6e94c1e9067"}]: dispatch 2024-04-02T04:53:24.234 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:23 smithi154 ceph-mon[35729]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84a2d337-6ac8-41e4-a922-a6e94c1e9067"}]: dispatch 2024-04-02T04:53:24.234 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:23 smithi154 ceph-mon[35729]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "84a2d337-6ac8-41e4-a922-a6e94c1e9067"}]': finished 2024-04-02T04:53:24.234 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:23 smithi154 ceph-mon[35729]: osdmap e23: 4 total, 3 up, 4 in 2024-04-02T04:53:24.234 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:23 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:53:24.256 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 02 04:53:24 smithi152 sudo[68533]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T04:53:24.257 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 02 04:53:24 smithi152 sudo[68533]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-02T04:53:24.257 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 02 04:53:24 smithi152 sudo[68533]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-02T04:53:24.257 INFO:journalctl@ceph.osd.2.smithi152.stdout:Apr 02 04:53:24 smithi152 sudo[68533]: pam_unix(sudo:session): session closed for user root 2024-04-02T04:53:24.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:23 smithi152 ceph-mon[29073]: Detected new or changed devices on smithi152 2024-04-02T04:53:24.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:23 smithi152 ceph-mon[29073]: Adjusting osd_memory_target on smithi152 to 4288M 2024-04-02T04:53:24.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:23 smithi152 ceph-mon[29073]: osdmap e22: 3 total, 3 up, 3 in 2024-04-02T04:53:24.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:23 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:24.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:23 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:53:24.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:23 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:53:24.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:23 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:24.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:23 smithi152 ceph-mon[29073]: from='client.? 172.21.15.154:0/3731552346' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84a2d337-6ac8-41e4-a922-a6e94c1e9067"}]: dispatch 2024-04-02T04:53:24.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:23 smithi152 ceph-mon[29073]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "84a2d337-6ac8-41e4-a922-a6e94c1e9067"}]: dispatch 2024-04-02T04:53:24.258 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:23 smithi152 ceph-mon[29073]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "84a2d337-6ac8-41e4-a922-a6e94c1e9067"}]': finished 2024-04-02T04:53:24.258 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:23 smithi152 ceph-mon[29073]: osdmap e23: 4 total, 3 up, 4 in 2024-04-02T04:53:24.258 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:23 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:53:24.595 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:24 smithi152 sudo[68536]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T04:53:24.595 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:24 smithi152 sudo[68536]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-02T04:53:24.596 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:24 smithi152 sudo[68536]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-02T04:53:24.596 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:24 smithi152 sudo[68536]: pam_unix(sudo:session): session closed for user root 2024-04-02T04:53:25.006 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:24 smithi152 sudo[68539]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T04:53:25.006 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:24 smithi152 sudo[68539]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-02T04:53:25.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:24 smithi152 sudo[68539]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-02T04:53:25.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:24 smithi152 sudo[68539]: pam_unix(sudo:session): session closed for user root 2024-04-02T04:53:25.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:24 smithi154 sudo[39335]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-02T04:53:25.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:24 smithi154 sudo[39335]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-02T04:53:25.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:24 smithi154 sudo[39335]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-02T04:53:25.098 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:24 smithi154 sudo[39335]: pam_unix(sudo:session): session closed for user root 2024-04-02T04:53:25.112 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:25.108+0000 7f3a7dffb700 1 -- 172.21.15.154:0/611754367 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f3a80053560 con 0x7f3a8c0fe0d0 2024-04-02T04:53:25.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:25 smithi154 sudo[39526]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-02T04:53:25.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:25 smithi154 sudo[39526]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-04-02T04:53:25.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:25 smithi154 sudo[39526]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-04-02T04:53:25.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:25 smithi154 sudo[39526]: pam_unix(sudo:session): session closed for user root 2024-04-02T04:53:25.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:25 smithi154 ceph-mon[35729]: pgmap v78: 1 pgs: 1 creating+activating; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-02T04:53:25.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:25 smithi154 ceph-mon[35729]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T04:53:25.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:25 smithi154 ceph-mon[35729]: from='client.? 172.21.15.154:0/3581033784' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:53:25.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:25 smithi154 ceph-mon[35729]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T04:53:25.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:25 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi152"}]: dispatch 2024-04-02T04:53:25.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:25 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-04-02T04:53:25.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:25 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi152"}]: dispatch 2024-04-02T04:53:25.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:25 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-04-02T04:53:25.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:25 smithi152 ceph-mon[29073]: pgmap v78: 1 pgs: 1 creating+activating; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-04-02T04:53:25.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:25 smithi152 ceph-mon[29073]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T04:53:25.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:25 smithi152 ceph-mon[29073]: from='client.? 172.21.15.154:0/3581033784' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:53:25.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:25 smithi152 ceph-mon[29073]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T04:53:25.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:25 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi152"}]: dispatch 2024-04-02T04:53:25.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:25 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-04-02T04:53:25.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:25 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi152"}]: dispatch 2024-04-02T04:53:25.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:25 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mon metadata", "id": "smithi154"}]: dispatch 2024-04-02T04:53:26.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:26 smithi154 ceph-mon[35729]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T04:53:26.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:26 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:53:26.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:26 smithi154 ceph-mon[35729]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T04:53:26.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:26 smithi154 ceph-mon[35729]: mgrmap e19: smithi152.jivdcl(active, since 2m), standbys: smithi154.pplnxz 2024-04-02T04:53:26.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:26 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:26.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:26 smithi152 ceph-mon[29073]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-02T04:53:26.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:26 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:53:26.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:26 smithi152 ceph-mon[29073]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-02T04:53:26.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:26 smithi152 ceph-mon[29073]: mgrmap e19: smithi152.jivdcl(active, since 2m), standbys: smithi154.pplnxz 2024-04-02T04:53:26.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:26 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:27.391 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:27 smithi154 ceph-mon[35729]: pgmap v79: 1 pgs: 1 creating+activating; 0 B data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-02T04:53:27.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:27 smithi152 ceph-mon[29073]: pgmap v79: 1 pgs: 1 creating+activating; 0 B data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-02T04:53:29.245 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:29 smithi154 ceph-mon[35729]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-02T04:53:29.426 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:29 smithi152 ceph-mon[29073]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-04-02T04:53:31.312 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:31 smithi154 ceph-mon[35729]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-02T04:53:31.312 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:31 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-02T04:53:31.312 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:31 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:53:31.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:31 smithi152 ceph-mon[29073]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-02T04:53:31.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:31 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-02T04:53:31.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:31 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:53:32.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:32 smithi154 ceph-mon[35729]: Deploying daemon osd.3 on smithi154 2024-04-02T04:53:32.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:32 smithi152 ceph-mon[29073]: Deploying daemon osd.3 on smithi154 2024-04-02T04:53:33.438 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:33 smithi154 ceph-mon[35729]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-02T04:53:33.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:33 smithi152 ceph-mon[29073]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-02T04:53:35.399 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:35 smithi154 ceph-mon[35729]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-02T04:53:35.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:35 smithi152 ceph-mon[29073]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-02T04:53:36.898 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:36 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:36.898 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:36 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:36.898 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:36 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:53:36.898 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:36 smithi154 ceph-mon[35729]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-02T04:53:37.006 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:36 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:37.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:36 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:37.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:36 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:53:37.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:36 smithi152 ceph-mon[29073]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-02T04:53:38.300 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:38.296+0000 7f3a7dffb700 1 -- 172.21.15.154:0/611754367 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7f3a8c10f8e0 con 0x7f3a78069210 2024-04-02T04:53:38.300 INFO:teuthology.orchestra.run.smithi154.stdout:Created osd(s) 3 on host 'smithi154' 2024-04-02T04:53:38.302 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:38.299+0000 7f3a93630700 1 -- 172.21.15.154:0/611754367 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f3a78069210 msgr2=0x7f3a7806b6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:53:38.302 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:38.299+0000 7f3a93630700 1 --2- 172.21.15.154:0/611754367 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f3a78069210 0x7f3a7806b6c0 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 crypto rx=0x7f3a88009fd0 tx=0x7f3a88009e30 comp rx=0 tx=0).stop 2024-04-02T04:53:38.303 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:38.299+0000 7f3a93630700 1 -- 172.21.15.154:0/611754367 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3a8c0fe0d0 msgr2=0x7f3a8c10df40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:53:38.303 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:38.299+0000 7f3a93630700 1 --2- 172.21.15.154:0/611754367 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3a8c0fe0d0 0x7f3a8c10df40 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 crypto rx=0x7f3a80002a50 tx=0x7f3a800046b0 comp rx=0 tx=0).stop 2024-04-02T04:53:38.303 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:38.299+0000 7f3a93630700 1 -- 172.21.15.154:0/611754367 shutdown_connections 2024-04-02T04:53:38.303 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:38.299+0000 7f3a93630700 1 --2- 172.21.15.154:0/611754367 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f3a78069210 0x7f3a7806b6c0 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:53:38.303 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:38.299+0000 7f3a93630700 1 --2- 172.21.15.154:0/611754367 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3a8c0fcee0 0x7f3a8c10da00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:53:38.303 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:38.299+0000 7f3a93630700 1 --2- 172.21.15.154:0/611754367 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3a8c0fe0d0 0x7f3a8c10df40 unknown :-1 s=CLOSED pgs=5 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:53:38.303 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:38.299+0000 7f3a93630700 1 -- 172.21.15.154:0/611754367 >> 172.21.15.154:0/611754367 conn(0x7f3a8c0f8560 msgr2=0x7f3a8c0f9bf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:53:38.303 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:38.299+0000 7f3a93630700 1 -- 172.21.15.154:0/611754367 shutdown_connections 2024-04-02T04:53:38.303 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:38.299+0000 7f3a93630700 1 -- 172.21.15.154:0/611754367 wait complete. 2024-04-02T04:53:39.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:39 smithi152 ceph-mon[29073]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-02T04:53:39.530 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:39 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:39.530 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:39 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:39.538 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:39 smithi154 ceph-mon[35729]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-02T04:53:39.538 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:39 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:39.538 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:39 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:39.563 DEBUG:teuthology.orchestra.run.smithi154:osd.3> sudo journalctl -f -n 0 -u ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4@osd.3.service 2024-04-02T04:53:39.565 INFO:tasks.cephadm:Deploying osd.4 on smithi154 with /dev/vg_nvme/lv_3... 2024-04-02T04:53:39.565 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-02T04:53:39.945 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi154/config 2024-04-02T04:53:40.402 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:40 smithi154 ceph-mon[35729]: from='osd.3 [v2:172.21.15.154:6800/3366219775,v1:172.21.15.154:6801/3366219775]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-02T04:53:40.402 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:40 smithi154 ceph-mon[35729]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-02T04:53:40.402 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:40 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:53:40.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:40 smithi152 ceph-mon[29073]: from='osd.3 [v2:172.21.15.154:6800/3366219775,v1:172.21.15.154:6801/3366219775]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-02T04:53:40.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:40 smithi152 ceph-mon[29073]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-02T04:53:40.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:40 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:53:41.316 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:41 smithi154 ceph-mon[35729]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-02T04:53:41.317 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:41 smithi154 ceph-mon[35729]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-02T04:53:41.317 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:41 smithi154 ceph-mon[35729]: osdmap e24: 4 total, 3 up, 4 in 2024-04-02T04:53:41.317 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:41 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:53:41.317 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:41 smithi154 ceph-mon[35729]: from='osd.3 [v2:172.21.15.154:6800/3366219775,v1:172.21.15.154:6801/3366219775]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-04-02T04:53:41.317 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:41 smithi154 ceph-mon[35729]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-04-02T04:53:41.317 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:41 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:41.317 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:41 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:41.317 INFO:journalctl@ceph.osd.3.smithi154.stdout:Apr 02 04:53:41 smithi154 ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-osd-3[41292]: 2024-04-02T04:53:41.267+0000 7ff38be8a700 -1 osd.3 0 waiting for initial osdmap 2024-04-02T04:53:41.317 INFO:journalctl@ceph.osd.3.smithi154.stdout:Apr 02 04:53:41 smithi154 ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-osd-3[41292]: 2024-04-02T04:53:41.272+0000 7ff386484700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T04:53:41.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:41 smithi152 ceph-mon[29073]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-02T04:53:41.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:41 smithi152 ceph-mon[29073]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-02T04:53:41.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:41 smithi152 ceph-mon[29073]: osdmap e24: 4 total, 3 up, 4 in 2024-04-02T04:53:41.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:41 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:53:41.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:41 smithi152 ceph-mon[29073]: from='osd.3 [v2:172.21.15.154:6800/3366219775,v1:172.21.15.154:6801/3366219775]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-04-02T04:53:41.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:41 smithi152 ceph-mon[29073]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-04-02T04:53:41.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:41 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:41.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:41 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:42.348 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:42 smithi154 ceph-mon[35729]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-04-02T04:53:42.349 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:42 smithi154 ceph-mon[35729]: osdmap e25: 4 total, 3 up, 4 in 2024-04-02T04:53:42.349 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:42 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:53:42.349 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:42 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:53:42.503 INFO:teuthology.orchestra.run.smithi154.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-02T04:53:42.503 INFO:teuthology.orchestra.run.smithi154.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-02T04:53:42.503 INFO:teuthology.orchestra.run.smithi154.stderr: stderr: 10+0 records in 2024-04-02T04:53:42.503 INFO:teuthology.orchestra.run.smithi154.stderr:10+0 records out 2024-04-02T04:53:42.503 INFO:teuthology.orchestra.run.smithi154.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0121597 s, 862 MB/s 2024-04-02T04:53:42.503 INFO:teuthology.orchestra.run.smithi154.stderr:--> Zapping successful for: 2024-04-02T04:53:42.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:42 smithi152 ceph-mon[29073]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-04-02T04:53:42.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:42 smithi152 ceph-mon[29073]: osdmap e25: 4 total, 3 up, 4 in 2024-04-02T04:53:42.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:42 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:53:42.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:42 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:53:43.136 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi154:vg_nvme/lv_3 2024-04-02T04:53:43.423 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:43 smithi154 ceph-mon[35729]: purged_snaps scrub starts 2024-04-02T04:53:43.423 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:43 smithi154 ceph-mon[35729]: purged_snaps scrub ok 2024-04-02T04:53:43.423 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:43 smithi154 ceph-mon[35729]: pgmap v89: 1 pgs: 1 peering; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-02T04:53:43.424 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:43 smithi154 ceph-mon[35729]: osd.3 [v2:172.21.15.154:6800/3366219775,v1:172.21.15.154:6801/3366219775] boot 2024-04-02T04:53:43.424 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:43 smithi154 ceph-mon[35729]: osdmap e26: 4 total, 4 up, 4 in 2024-04-02T04:53:43.424 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:43 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:53:43.692 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi154/config 2024-04-02T04:53:43.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:43 smithi152 ceph-mon[29073]: purged_snaps scrub starts 2024-04-02T04:53:43.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:43 smithi152 ceph-mon[29073]: purged_snaps scrub ok 2024-04-02T04:53:43.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:43 smithi152 ceph-mon[29073]: pgmap v89: 1 pgs: 1 peering; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-04-02T04:53:43.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:43 smithi152 ceph-mon[29073]: osd.3 [v2:172.21.15.154:6800/3366219775,v1:172.21.15.154:6801/3366219775] boot 2024-04-02T04:53:43.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:43 smithi152 ceph-mon[29073]: osdmap e26: 4 total, 4 up, 4 in 2024-04-02T04:53:43.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:43 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-02T04:53:44.398 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:44 smithi154 ceph-mon[35729]: osdmap e27: 4 total, 4 up, 4 in 2024-04-02T04:53:44.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:44 smithi152 ceph-mon[29073]: osdmap e27: 4 total, 4 up, 4 in 2024-04-02T04:53:45.540 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:45 smithi154 ceph-mon[35729]: pgmap v92: 1 pgs: 1 peering; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-02T04:53:45.540 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:45 smithi154 ceph-mon[35729]: osdmap e28: 4 total, 4 up, 4 in 2024-04-02T04:53:45.541 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:45 smithi154 ceph-mon[35729]: osdmap e29: 4 total, 4 up, 4 in 2024-04-02T04:53:45.541 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:45 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:45.541 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:45 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:45.541 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:45 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:53:45.541 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:45 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:45.541 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:45 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:53:45.541 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:45 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:53:45.541 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:45 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:45.541 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:45 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:53:45.541 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:45 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:53:45.541 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:45 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:53:45.541 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:45 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:45.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:45 smithi152 ceph-mon[29073]: pgmap v92: 1 pgs: 1 peering; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-02T04:53:45.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:45 smithi152 ceph-mon[29073]: osdmap e28: 4 total, 4 up, 4 in 2024-04-02T04:53:45.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:45 smithi152 ceph-mon[29073]: osdmap e29: 4 total, 4 up, 4 in 2024-04-02T04:53:45.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:45 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:45.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:45 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:45.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:45 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:53:45.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:45 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:45.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:45 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:53:45.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:45 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:53:45.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:45 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:45.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:45 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:53:45.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:45 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:53:45.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:45 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:53:45.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:45 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:53:45.863 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.859+0000 7fd469b9f700 1 -- 172.21.15.154:0/3446101677 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4640fdff0 msgr2=0x7fd4640fe460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:53:45.863 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.859+0000 7fd469b9f700 1 --2- 172.21.15.154:0/3446101677 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4640fdff0 0x7fd4640fe460 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 crypto rx=0x7fd458004940 tx=0x7fd458009e30 comp rx=0 tx=0).stop 2024-04-02T04:53:45.863 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.859+0000 7fd469b9f700 1 -- 172.21.15.154:0/3446101677 shutdown_connections 2024-04-02T04:53:45.863 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.859+0000 7fd469b9f700 1 --2- 172.21.15.154:0/3446101677 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4640fdff0 0x7fd4640fe460 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:53:45.863 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.859+0000 7fd469b9f700 1 --2- 172.21.15.154:0/3446101677 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd4640fce80 0x7fd4640fd290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:53:45.863 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.859+0000 7fd469b9f700 1 -- 172.21.15.154:0/3446101677 >> 172.21.15.154:0/3446101677 conn(0x7fd4640f8580 msgr2=0x7fd4640fa990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:53:45.863 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.859+0000 7fd469b9f700 1 -- 172.21.15.154:0/3446101677 shutdown_connections 2024-04-02T04:53:45.863 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.859+0000 7fd469b9f700 1 -- 172.21.15.154:0/3446101677 wait complete. 2024-04-02T04:53:45.863 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.860+0000 7fd469b9f700 1 Processor -- start 2024-04-02T04:53:45.865 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.860+0000 7fd469b9f700 1 -- start start 2024-04-02T04:53:45.865 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.860+0000 7fd469b9f700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4640fce80 0x7fd46410da20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:53:45.865 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.860+0000 7fd469b9f700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd4640fdff0 0x7fd46410df60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:53:45.865 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.860+0000 7fd469b9f700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd46410e560 con 0x7fd4640fdff0 2024-04-02T04:53:45.865 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.860+0000 7fd469b9f700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd46410edb0 con 0x7fd4640fce80 2024-04-02T04:53:45.865 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.860+0000 7fd4637fe700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4640fce80 0x7fd46410da20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:53:45.865 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.860+0000 7fd4637fe700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4640fce80 0x7fd46410da20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.154:49396/0 (socket says 172.21.15.154:49396) 2024-04-02T04:53:45.866 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.860+0000 7fd4637fe700 1 -- 172.21.15.154:0/3649891896 learned_addr learned my addr 172.21.15.154:0/3649891896 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-04-02T04:53:45.866 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.861+0000 7fd462ffd700 1 --2- 172.21.15.154:0/3649891896 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd4640fdff0 0x7fd46410df60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:53:45.866 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.861+0000 7fd4637fe700 1 -- 172.21.15.154:0/3649891896 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd4640fdff0 msgr2=0x7fd46410df60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:53:45.866 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.861+0000 7fd4637fe700 1 --2- 172.21.15.154:0/3649891896 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd4640fdff0 0x7fd46410df60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:53:45.866 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.861+0000 7fd4637fe700 1 -- 172.21.15.154:0/3649891896 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd458005040 con 0x7fd4640fce80 2024-04-02T04:53:45.866 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.861+0000 7fd4637fe700 1 --2- 172.21.15.154:0/3649891896 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4640fce80 0x7fd46410da20 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7fd45400cac0 tx=0x7fd454007ee0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:53:45.866 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.861+0000 7fd44bfff700 1 -- 172.21.15.154:0/3649891896 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fd45400aa50 con 0x7fd4640fce80 2024-04-02T04:53:45.866 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.861+0000 7fd44bfff700 1 -- 172.21.15.154:0/3649891896 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fd45400abb0 con 0x7fd4640fce80 2024-04-02T04:53:45.866 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.861+0000 7fd469b9f700 1 -- 172.21.15.154:0/3649891896 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd46410f090 con 0x7fd4640fce80 2024-04-02T04:53:45.866 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.861+0000 7fd44bfff700 1 -- 172.21.15.154:0/3649891896 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fd4540115e0 con 0x7fd4640fce80 2024-04-02T04:53:45.866 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.861+0000 7fd469b9f700 1 -- 172.21.15.154:0/3649891896 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd46410f5d0 con 0x7fd4640fce80 2024-04-02T04:53:45.866 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.862+0000 7fd469b9f700 1 -- 172.21.15.154:0/3649891896 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd46405f5f0 con 0x7fd4640fce80 2024-04-02T04:53:45.867 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.862+0000 7fd44bfff700 1 -- 172.21.15.154:0/3649891896 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7fd45400ad20 con 0x7fd4640fce80 2024-04-02T04:53:45.867 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.863+0000 7fd44bfff700 1 --2- 172.21.15.154:0/3649891896 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd44c0692a0 0x7fd44c06b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:53:45.867 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.863+0000 7fd44bfff700 1 -- 172.21.15.154:0/3649891896 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(30..30 src has 1..30) v4 ==== 3770+0+0 (secure 0 0 0) 0x7fd454087280 con 0x7fd4640fce80 2024-04-02T04:53:45.867 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.863+0000 7fd462ffd700 1 --2- 172.21.15.154:0/3649891896 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd44c0692a0 0x7fd44c06b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:53:45.867 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.864+0000 7fd462ffd700 1 --2- 172.21.15.154:0/3649891896 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd44c0692a0 0x7fd44c06b750 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fd458000c00 tx=0x7fd458009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:53:45.870 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:45.867+0000 7fd44bfff700 1 -- 172.21.15.154:0/3649891896 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fd454057660 con 0x7fd4640fce80 2024-04-02T04:53:46.062 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:53:46.058+0000 7fd469b9f700 1 -- 172.21.15.154:0/3649891896 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_3", "target": ["mon-mgr", ""]}) v1 -- 0x7fd46410f8f0 con 0x7fd44c0692a0 2024-04-02T04:53:46.611 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:46 smithi154 ceph-mon[35729]: Detected new or changed devices on smithi154 2024-04-02T04:53:46.611 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:46 smithi154 ceph-mon[35729]: Adjusting osd_memory_target on smithi154 to 15936M 2024-04-02T04:53:46.611 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:46 smithi154 ceph-mon[35729]: osdmap e30: 4 total, 4 up, 4 in 2024-04-02T04:53:46.611 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:46 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:53:46.611 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:46 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:53:46.611 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:46 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:53:46.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:46 smithi152 ceph-mon[29073]: Detected new or changed devices on smithi154 2024-04-02T04:53:46.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:46 smithi152 ceph-mon[29073]: Adjusting osd_memory_target on smithi154 to 15936M 2024-04-02T04:53:46.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:46 smithi152 ceph-mon[29073]: osdmap e30: 4 total, 4 up, 4 in 2024-04-02T04:53:46.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:46 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:53:46.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:46 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:53:46.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:46 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:53:47.560 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:47 smithi154 ceph-mon[35729]: pgmap v96: 1 pgs: 1 peering; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-02T04:53:47.560 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:47 smithi154 ceph-mon[35729]: from='client.24159 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:53:47.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:47 smithi152 ceph-mon[29073]: pgmap v96: 1 pgs: 1 peering; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-02T04:53:47.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:47 smithi152 ceph-mon[29073]: from='client.24159 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:53:49.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:49 smithi154 ceph-mon[35729]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail; 123 KiB/s, 0 objects/s recovering 2024-04-02T04:53:49.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:49 smithi154 ceph-mon[35729]: from='client.? 172.21.15.154:0/3782552692' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d547308c-898c-4165-9762-c8fd4170dc39"}]: dispatch 2024-04-02T04:53:49.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:49 smithi154 ceph-mon[35729]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d547308c-898c-4165-9762-c8fd4170dc39"}]: dispatch 2024-04-02T04:53:49.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:49 smithi154 ceph-mon[35729]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d547308c-898c-4165-9762-c8fd4170dc39"}]': finished 2024-04-02T04:53:49.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:49 smithi154 ceph-mon[35729]: osdmap e31: 5 total, 4 up, 5 in 2024-04-02T04:53:49.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:49 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:53:49.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:49 smithi154 ceph-mon[35729]: from='client.? 172.21.15.154:0/1068676986' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:53:49.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:49 smithi152 ceph-mon[29073]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail; 123 KiB/s, 0 objects/s recovering 2024-04-02T04:53:49.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:49 smithi152 ceph-mon[29073]: from='client.? 172.21.15.154:0/3782552692' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d547308c-898c-4165-9762-c8fd4170dc39"}]: dispatch 2024-04-02T04:53:49.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:49 smithi152 ceph-mon[29073]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d547308c-898c-4165-9762-c8fd4170dc39"}]: dispatch 2024-04-02T04:53:49.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:49 smithi152 ceph-mon[29073]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d547308c-898c-4165-9762-c8fd4170dc39"}]': finished 2024-04-02T04:53:49.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:49 smithi152 ceph-mon[29073]: osdmap e31: 5 total, 4 up, 5 in 2024-04-02T04:53:49.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:49 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:53:49.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:49 smithi152 ceph-mon[29073]: from='client.? 172.21.15.154:0/1068676986' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:53:51.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:51 smithi154 ceph-mon[35729]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail; 101 KiB/s, 0 objects/s recovering 2024-04-02T04:53:51.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:51 smithi152 ceph-mon[29073]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail; 101 KiB/s, 0 objects/s recovering 2024-04-02T04:53:53.571 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:53 smithi154 ceph-mon[35729]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-04-02T04:53:53.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:53 smithi152 ceph-mon[29073]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-04-02T04:53:55.468 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:55 smithi154 ceph-mon[35729]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-04-02T04:53:55.468 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:55 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:53:55.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:55 smithi152 ceph-mon[29073]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail; 69 KiB/s, 0 objects/s recovering 2024-04-02T04:53:55.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:55 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:53:56.561 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:56 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-02T04:53:56.561 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:56 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:53:56.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:56 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-02T04:53:56.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:56 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:53:57.453 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:57 smithi154 ceph-mon[35729]: Deploying daemon osd.4 on smithi154 2024-04-02T04:53:57.454 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:57 smithi154 ceph-mon[35729]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-02T04:53:57.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:57 smithi152 ceph-mon[29073]: Deploying daemon osd.4 on smithi154 2024-04-02T04:53:57.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:57 smithi152 ceph-mon[29073]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-02T04:53:59.711 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:53:59 smithi154 ceph-mon[35729]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-02T04:53:59.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:53:59 smithi152 ceph-mon[29073]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-02T04:54:01.544 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:01 smithi154 ceph-mon[35729]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-02T04:54:01.544 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:01 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:01.544 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:01 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:01.544 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:01 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:54:01.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:01 smithi152 ceph-mon[29073]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-02T04:54:01.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:01 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:01.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:01 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:01.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:01 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:54:03.474 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:03 smithi154 ceph-mon[35729]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-02T04:54:03.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:03 smithi152 ceph-mon[29073]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-02T04:54:03.823 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:03.819+0000 7fd44bfff700 1 -- 172.21.15.154:0/3649891896 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7fd46410f8f0 con 0x7fd44c0692a0 2024-04-02T04:54:03.823 INFO:teuthology.orchestra.run.smithi154.stdout:Created osd(s) 4 on host 'smithi154' 2024-04-02T04:54:03.825 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:03.822+0000 7fd469b9f700 1 -- 172.21.15.154:0/3649891896 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd44c0692a0 msgr2=0x7fd44c06b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:03.825 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:03.822+0000 7fd469b9f700 1 --2- 172.21.15.154:0/3649891896 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd44c0692a0 0x7fd44c06b750 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7fd458000c00 tx=0x7fd458009e30 comp rx=0 tx=0).stop 2024-04-02T04:54:03.825 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:03.822+0000 7fd469b9f700 1 -- 172.21.15.154:0/3649891896 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4640fce80 msgr2=0x7fd46410da20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:03.825 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:03.822+0000 7fd469b9f700 1 --2- 172.21.15.154:0/3649891896 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4640fce80 0x7fd46410da20 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 crypto rx=0x7fd45400cac0 tx=0x7fd454007ee0 comp rx=0 tx=0).stop 2024-04-02T04:54:03.825 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:03.822+0000 7fd469b9f700 1 -- 172.21.15.154:0/3649891896 shutdown_connections 2024-04-02T04:54:03.825 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:03.822+0000 7fd469b9f700 1 --2- 172.21.15.154:0/3649891896 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd44c0692a0 0x7fd44c06b750 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:03.825 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:03.822+0000 7fd469b9f700 1 --2- 172.21.15.154:0/3649891896 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd4640fce80 0x7fd46410da20 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:03.825 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:03.822+0000 7fd469b9f700 1 --2- 172.21.15.154:0/3649891896 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd4640fdff0 0x7fd46410df60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:03.826 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:03.822+0000 7fd469b9f700 1 -- 172.21.15.154:0/3649891896 >> 172.21.15.154:0/3649891896 conn(0x7fd4640f8580 msgr2=0x7fd4640f9c70 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:54:03.826 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:03.822+0000 7fd469b9f700 1 -- 172.21.15.154:0/3649891896 shutdown_connections 2024-04-02T04:54:03.826 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:03.822+0000 7fd469b9f700 1 -- 172.21.15.154:0/3649891896 wait complete. 2024-04-02T04:54:04.419 DEBUG:teuthology.orchestra.run.smithi154:osd.4> sudo journalctl -f -n 0 -u ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4@osd.4.service 2024-04-02T04:54:04.421 INFO:tasks.cephadm:Deploying osd.5 on smithi154 with /dev/vg_nvme/lv_2... 2024-04-02T04:54:04.421 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-02T04:54:04.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:04 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:04.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:04 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:04.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:04 smithi154 ceph-mon[35729]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-02T04:54:04.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:04 smithi154 ceph-mon[35729]: from='osd.4 [v2:172.21.15.154:6808/220818867,v1:172.21.15.154:6809/220818867]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T04:54:04.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:04 smithi154 ceph-mon[35729]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T04:54:04.984 INFO:journalctl@ceph.osd.4.smithi154.stdout:Apr 02 04:54:04 smithi154 ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-osd-4[47872]: 2024-04-02T04:54:04.743+0000 7ff40a1c5540 -1 osd.4 0 log_to_monitors true 2024-04-02T04:54:05.162 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi154/config 2024-04-02T04:54:05.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:04 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:05.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:04 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:05.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:04 smithi152 ceph-mon[29073]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-02T04:54:05.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:04 smithi152 ceph-mon[29073]: from='osd.4 [v2:172.21.15.154:6808/220818867,v1:172.21.15.154:6809/220818867]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T04:54:05.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:04 smithi152 ceph-mon[29073]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-02T04:54:06.164 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:05 smithi154 ceph-mon[35729]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-02T04:54:06.164 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:05 smithi154 ceph-mon[35729]: osdmap e32: 5 total, 4 up, 5 in 2024-04-02T04:54:06.165 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:05 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:54:06.165 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:05 smithi154 ceph-mon[35729]: from='osd.4 [v2:172.21.15.154:6808/220818867,v1:172.21.15.154:6809/220818867]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-04-02T04:54:06.165 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:05 smithi154 ceph-mon[35729]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-04-02T04:54:06.165 INFO:journalctl@ceph.osd.4.smithi154.stdout:Apr 02 04:54:05 smithi154 ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-osd-4[47872]: 2024-04-02T04:54:05.836+0000 7ff400997700 -1 osd.4 0 waiting for initial osdmap 2024-04-02T04:54:06.165 INFO:journalctl@ceph.osd.4.smithi154.stdout:Apr 02 04:54:05 smithi154 ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-osd-4[47872]: 2024-04-02T04:54:05.843+0000 7ff3fb792700 -1 osd.4 33 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T04:54:06.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:05 smithi152 ceph-mon[29073]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-02T04:54:06.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:05 smithi152 ceph-mon[29073]: osdmap e32: 5 total, 4 up, 5 in 2024-04-02T04:54:06.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:05 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:54:06.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:05 smithi152 ceph-mon[29073]: from='osd.4 [v2:172.21.15.154:6808/220818867,v1:172.21.15.154:6809/220818867]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-04-02T04:54:06.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:05 smithi152 ceph-mon[29073]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-04-02T04:54:07.054 INFO:teuthology.orchestra.run.smithi154.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-02T04:54:07.054 INFO:teuthology.orchestra.run.smithi154.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-02T04:54:07.054 INFO:teuthology.orchestra.run.smithi154.stderr: stderr: 10+0 records in 2024-04-02T04:54:07.054 INFO:teuthology.orchestra.run.smithi154.stderr:10+0 records out 2024-04-02T04:54:07.055 INFO:teuthology.orchestra.run.smithi154.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0138482 s, 757 MB/s 2024-04-02T04:54:07.055 INFO:teuthology.orchestra.run.smithi154.stderr:--> Zapping successful for: 2024-04-02T04:54:07.162 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:06 smithi154 ceph-mon[35729]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-04-02T04:54:07.162 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:06 smithi154 ceph-mon[35729]: osdmap e33: 5 total, 4 up, 5 in 2024-04-02T04:54:07.162 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:06 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:54:07.162 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:06 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:54:07.162 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:06 smithi154 ceph-mon[35729]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-02T04:54:07.162 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:06 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:07.163 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:06 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:07.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:06 smithi152 ceph-mon[29073]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-04-02T04:54:07.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:06 smithi152 ceph-mon[29073]: osdmap e33: 5 total, 4 up, 5 in 2024-04-02T04:54:07.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:06 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:54:07.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:06 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:54:07.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:06 smithi152 ceph-mon[29073]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-04-02T04:54:07.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:06 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:07.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:06 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:07.969 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:07 smithi154 ceph-mon[35729]: purged_snaps scrub starts 2024-04-02T04:54:07.969 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:07 smithi154 ceph-mon[35729]: purged_snaps scrub ok 2024-04-02T04:54:07.969 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:07 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:54:07.969 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:07 smithi154 ceph-mon[35729]: osd.4 [v2:172.21.15.154:6808/220818867,v1:172.21.15.154:6809/220818867] boot 2024-04-02T04:54:07.970 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:07 smithi154 ceph-mon[35729]: osdmap e34: 5 total, 5 up, 5 in 2024-04-02T04:54:07.970 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:07 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:54:08.134 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:07 smithi152 ceph-mon[29073]: purged_snaps scrub starts 2024-04-02T04:54:08.134 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:07 smithi152 ceph-mon[29073]: purged_snaps scrub ok 2024-04-02T04:54:08.134 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:07 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:54:08.134 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:07 smithi152 ceph-mon[29073]: osd.4 [v2:172.21.15.154:6808/220818867,v1:172.21.15.154:6809/220818867] boot 2024-04-02T04:54:08.134 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:07 smithi152 ceph-mon[29073]: osdmap e34: 5 total, 5 up, 5 in 2024-04-02T04:54:08.134 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:07 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-02T04:54:08.347 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph orch daemon add osd smithi154:vg_nvme/lv_2 2024-04-02T04:54:08.767 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi154/config 2024-04-02T04:54:09.111 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:08 smithi154 ceph-mon[35729]: osdmap e35: 5 total, 5 up, 5 in 2024-04-02T04:54:09.111 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:08 smithi154 ceph-mon[35729]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:09.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:08 smithi152 ceph-mon[29073]: osdmap e35: 5 total, 5 up, 5 in 2024-04-02T04:54:09.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:08 smithi152 ceph-mon[29073]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:10.267 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:10 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:54:10.488 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.484+0000 7f65f5e82700 1 -- 172.21.15.154:0/3280380550 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f65f00fce40 msgr2=0x7f65f00fd250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:10.488 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.484+0000 7f65f5e82700 1 --2- 172.21.15.154:0/3280380550 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f65f00fce40 0x7f65f00fd250 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 crypto rx=0x7f65e0004a90 tx=0x7f65e0009dd0 comp rx=0 tx=0).stop 2024-04-02T04:54:10.488 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.484+0000 7f65f5e82700 1 -- 172.21.15.154:0/3280380550 shutdown_connections 2024-04-02T04:54:10.488 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.484+0000 7f65f5e82700 1 --2- 172.21.15.154:0/3280380550 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f65f00fe040 0x7f65f00fe4b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:10.488 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.484+0000 7f65f5e82700 1 --2- 172.21.15.154:0/3280380550 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f65f00fce40 0x7f65f00fd250 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:10.488 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.484+0000 7f65f5e82700 1 -- 172.21.15.154:0/3280380550 >> 172.21.15.154:0/3280380550 conn(0x7f65f00f8570 msgr2=0x7f65f00fa980 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:54:10.488 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.484+0000 7f65f5e82700 1 -- 172.21.15.154:0/3280380550 shutdown_connections 2024-04-02T04:54:10.488 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.484+0000 7f65f5e82700 1 -- 172.21.15.154:0/3280380550 wait complete. 2024-04-02T04:54:10.488 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.484+0000 7f65f5e82700 1 Processor -- start 2024-04-02T04:54:10.488 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.485+0000 7f65f5e82700 1 -- start start 2024-04-02T04:54:10.488 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.485+0000 7f65f5e82700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f65f00fce40 0x7f65f010da30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:10.488 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.485+0000 7f65f5e82700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f65f00fe040 0x7f65f010df70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:10.489 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.485+0000 7f65f5e82700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f65f010e600 con 0x7f65f00fe040 2024-04-02T04:54:10.489 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.485+0000 7f65f5e82700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f65f010edf0 con 0x7f65f00fce40 2024-04-02T04:54:10.490 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.485+0000 7f65ef7fe700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f65f00fce40 0x7f65f010da30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:10.490 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.485+0000 7f65ef7fe700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f65f00fce40 0x7f65f010da30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.154:33896/0 (socket says 172.21.15.154:33896) 2024-04-02T04:54:10.490 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.485+0000 7f65ef7fe700 1 -- 172.21.15.154:0/2444382589 learned_addr learned my addr 172.21.15.154:0/2444382589 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-04-02T04:54:10.490 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.485+0000 7f65eeffd700 1 --2- 172.21.15.154:0/2444382589 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f65f00fe040 0x7f65f010df70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:10.490 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.485+0000 7f65ef7fe700 1 -- 172.21.15.154:0/2444382589 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f65f00fe040 msgr2=0x7f65f010df70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:10.490 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.485+0000 7f65ef7fe700 1 --2- 172.21.15.154:0/2444382589 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f65f00fe040 0x7f65f010df70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:10.490 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.485+0000 7f65ef7fe700 1 -- 172.21.15.154:0/2444382589 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f65e0004650 con 0x7f65f00fce40 2024-04-02T04:54:10.490 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.485+0000 7f65ef7fe700 1 --2- 172.21.15.154:0/2444382589 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f65f00fce40 0x7f65f010da30 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f65e0009a00 tx=0x7f65e0009dd0 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:54:10.490 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.485+0000 7f65d7fff700 1 -- 172.21.15.154:0/2444382589 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f65e0019070 con 0x7f65f00fce40 2024-04-02T04:54:10.490 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.485+0000 7f65f5e82700 1 -- 172.21.15.154:0/2444382589 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f65f010f070 con 0x7f65f00fce40 2024-04-02T04:54:10.490 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.485+0000 7f65d7fff700 1 -- 172.21.15.154:0/2444382589 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f65e000bd00 con 0x7f65f00fce40 2024-04-02T04:54:10.491 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.485+0000 7f65d7fff700 1 -- 172.21.15.154:0/2444382589 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f65e000e040 con 0x7f65f00fce40 2024-04-02T04:54:10.491 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.486+0000 7f65f5e82700 1 -- 172.21.15.154:0/2444382589 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f65f010f550 con 0x7f65f00fce40 2024-04-02T04:54:10.491 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.486+0000 7f65f5e82700 1 -- 172.21.15.154:0/2444382589 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f65f005f5f0 con 0x7f65f00fce40 2024-04-02T04:54:10.491 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.487+0000 7f65d7fff700 1 -- 172.21.15.154:0/2444382589 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f65e0014a00 con 0x7f65f00fce40 2024-04-02T04:54:10.491 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.487+0000 7f65d7fff700 1 --2- 172.21.15.154:0/2444382589 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f65d80692a0 0x7f65d806b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:10.492 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.487+0000 7f65d7fff700 1 -- 172.21.15.154:0/2444382589 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(35..35 src has 1..35) v4 ==== 4202+0+0 (secure 0 0 0) 0x7f65e0087b90 con 0x7f65f00fce40 2024-04-02T04:54:10.492 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.488+0000 7f65eeffd700 1 --2- 172.21.15.154:0/2444382589 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f65d80692a0 0x7f65d806b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:10.492 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.488+0000 7f65eeffd700 1 --2- 172.21.15.154:0/2444382589 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f65d80692a0 0x7f65d806b750 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f65e4006fd0 tx=0x7f65e4008040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:54:10.495 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.492+0000 7f65d7fff700 1 -- 172.21.15.154:0/2444382589 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f65e0057e40 con 0x7f65f00fce40 2024-04-02T04:54:10.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:10 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:54:10.693 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:10.690+0000 7f65f5e82700 1 -- 172.21.15.154:0/2444382589 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_2", "target": ["mon-mgr", ""]}) v1 -- 0x7f65f010f810 con 0x7f65d80692a0 2024-04-02T04:54:11.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:11 smithi154 ceph-mon[35729]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:11.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:11 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:54:11.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:11 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:54:11.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:11 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:54:11.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:11 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:11.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:11 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:11.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:11 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:54:11.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:11 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:54:11.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:11 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:11.342 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:11 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:54:11.343 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:11 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:54:11.343 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:11 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:11.343 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:11 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:54:11.343 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:11 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:54:11.343 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:11 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:54:11.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:11 smithi152 ceph-mon[29073]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:11.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:11 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-02T04:54:11.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:11 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-02T04:54:11.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:11 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:54:11.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:11 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:11.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:11 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:11.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:11 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:54:11.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:11 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:54:11.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:11 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:11.508 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:11 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:54:11.508 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:11 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:54:11.508 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:11 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:11.508 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:11 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:54:11.508 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:11 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:54:11.508 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:11 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:54:12.373 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:12 smithi154 ceph-mon[35729]: from='client.24177 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:54:12.374 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:12 smithi154 ceph-mon[35729]: Detected new or changed devices on smithi154 2024-04-02T04:54:12.374 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:12 smithi154 ceph-mon[35729]: Adjusting osd_memory_target on smithi154 to 7968M 2024-04-02T04:54:12.374 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:12 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:12.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:12 smithi152 ceph-mon[29073]: from='client.24177 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi154:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:54:12.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:12 smithi152 ceph-mon[29073]: Detected new or changed devices on smithi154 2024-04-02T04:54:12.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:12 smithi152 ceph-mon[29073]: Adjusting osd_memory_target on smithi154 to 7968M 2024-04-02T04:54:12.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:12 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:13.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:13 smithi154 ceph-mon[35729]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:13.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:13 smithi154 ceph-mon[35729]: from='client.? 172.21.15.154:0/2827042821' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c3d3760a-08a7-44f2-8288-ccc5d08df057"}]: dispatch 2024-04-02T04:54:13.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:13 smithi152 ceph-mon[29073]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:13.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:13 smithi152 ceph-mon[29073]: from='client.? 172.21.15.154:0/2827042821' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c3d3760a-08a7-44f2-8288-ccc5d08df057"}]: dispatch 2024-04-02T04:54:14.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:14 smithi154 ceph-mon[35729]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c3d3760a-08a7-44f2-8288-ccc5d08df057"}]: dispatch 2024-04-02T04:54:14.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:14 smithi154 ceph-mon[35729]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c3d3760a-08a7-44f2-8288-ccc5d08df057"}]': finished 2024-04-02T04:54:14.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:14 smithi154 ceph-mon[35729]: osdmap e36: 6 total, 5 up, 6 in 2024-04-02T04:54:14.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:14 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:54:14.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:14 smithi154 ceph-mon[35729]: from='client.? 172.21.15.154:0/1950573275' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:54:14.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:14 smithi152 ceph-mon[29073]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c3d3760a-08a7-44f2-8288-ccc5d08df057"}]: dispatch 2024-04-02T04:54:14.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:14 smithi152 ceph-mon[29073]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c3d3760a-08a7-44f2-8288-ccc5d08df057"}]': finished 2024-04-02T04:54:14.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:14 smithi152 ceph-mon[29073]: osdmap e36: 6 total, 5 up, 6 in 2024-04-02T04:54:14.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:14 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:54:14.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:14 smithi152 ceph-mon[29073]: from='client.? 172.21.15.154:0/1950573275' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-02T04:54:15.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:15 smithi154 ceph-mon[35729]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:15.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:15 smithi152 ceph-mon[29073]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:17.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:17 smithi154 ceph-mon[35729]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:17.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:17 smithi152 ceph-mon[29073]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:19.155 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:19 smithi154 ceph-mon[35729]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:19.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:19 smithi152 ceph-mon[29073]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:21.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:21 smithi154 ceph-mon[35729]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:21.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:21 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-02T04:54:21.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:21 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:54:21.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:21 smithi154 ceph-mon[35729]: Deploying daemon osd.5 on smithi154 2024-04-02T04:54:21.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:21 smithi152 ceph-mon[29073]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:21.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:21 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-02T04:54:21.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:21 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:54:21.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:21 smithi152 ceph-mon[29073]: Deploying daemon osd.5 on smithi154 2024-04-02T04:54:23.349 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:23 smithi154 ceph-mon[35729]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:23.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:23 smithi152 ceph-mon[29073]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:25.292 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:25 smithi154 ceph-mon[35729]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:25.292 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:25 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:54:25.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:25 smithi152 ceph-mon[29073]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:25.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:25 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:54:26.682 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:26 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:26.682 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:26 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:26.682 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:26 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:54:26.759 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:26 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:26.759 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:26 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:26.759 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:26 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:54:27.518 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:27 smithi154 ceph-mon[35729]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:27.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:27 smithi152 ceph-mon[29073]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:28.154 INFO:teuthology.orchestra.run.smithi154.stdout:Created osd(s) 5 on host 'smithi154' 2024-04-02T04:54:28.155 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:28.151+0000 7f65d7fff700 1 -- 172.21.15.154:0/2444382589 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+37 (secure 0 0 0) 0x7f65f010f810 con 0x7f65d80692a0 2024-04-02T04:54:28.157 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:28.153+0000 7f65f5e82700 1 -- 172.21.15.154:0/2444382589 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f65d80692a0 msgr2=0x7f65d806b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:28.157 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:28.153+0000 7f65f5e82700 1 --2- 172.21.15.154:0/2444382589 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f65d80692a0 0x7f65d806b750 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 crypto rx=0x7f65e4006fd0 tx=0x7f65e4008040 comp rx=0 tx=0).stop 2024-04-02T04:54:28.157 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:28.153+0000 7f65f5e82700 1 -- 172.21.15.154:0/2444382589 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f65f00fce40 msgr2=0x7f65f010da30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:28.157 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:28.153+0000 7f65f5e82700 1 --2- 172.21.15.154:0/2444382589 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f65f00fce40 0x7f65f010da30 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 crypto rx=0x7f65e0009a00 tx=0x7f65e0009dd0 comp rx=0 tx=0).stop 2024-04-02T04:54:28.158 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:28.154+0000 7f65f5e82700 1 -- 172.21.15.154:0/2444382589 shutdown_connections 2024-04-02T04:54:28.158 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:28.154+0000 7f65f5e82700 1 --2- 172.21.15.154:0/2444382589 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f65d80692a0 0x7f65d806b750 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:28.158 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:28.154+0000 7f65f5e82700 1 --2- 172.21.15.154:0/2444382589 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f65f00fce40 0x7f65f010da30 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:28.158 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:28.154+0000 7f65f5e82700 1 --2- 172.21.15.154:0/2444382589 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f65f00fe040 0x7f65f010df70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:28.158 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:28.154+0000 7f65f5e82700 1 -- 172.21.15.154:0/2444382589 >> 172.21.15.154:0/2444382589 conn(0x7f65f00f8570 msgr2=0x7f65f00f9bc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:54:28.158 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:28.154+0000 7f65f5e82700 1 -- 172.21.15.154:0/2444382589 shutdown_connections 2024-04-02T04:54:28.158 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:28.154+0000 7f65f5e82700 1 -- 172.21.15.154:0/2444382589 wait complete. 2024-04-02T04:54:29.012 DEBUG:teuthology.orchestra.run.smithi154:osd.5> sudo journalctl -f -n 0 -u ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4@osd.5.service 2024-04-02T04:54:29.015 INFO:tasks.cephadm:Waiting for 6 OSDs to come up... 2024-04-02T04:54:29.015 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph osd stat -f json 2024-04-02T04:54:29.283 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:54:29.317 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:29 smithi154 ceph-mon[35729]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:29.317 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:29 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:29.317 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:29 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:29.317 INFO:journalctl@ceph.osd.5.smithi154.stdout:Apr 02 04:54:29 smithi154 ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-osd-5[54772]: 2024-04-02T04:54:29.227+0000 7f10305cd540 -1 osd.5 0 log_to_monitors true 2024-04-02T04:54:29.405 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:29 smithi152 ceph-mon[29073]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:29.406 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:29 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:29.406 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:29 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:30.413 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:30 smithi152 ceph-mon[29073]: from='osd.5 [v2:172.21.15.154:6816/1178927498,v1:172.21.15.154:6817/1178927498]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T04:54:30.413 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:30 smithi152 ceph-mon[29073]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T04:54:30.413 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:30 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:30.413 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:30 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:30.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:30 smithi154 ceph-mon[35729]: from='osd.5 [v2:172.21.15.154:6816/1178927498,v1:172.21.15.154:6817/1178927498]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T04:54:30.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:30 smithi154 ceph-mon[35729]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-02T04:54:30.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:30 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:30.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:30 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:30.945 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.939+0000 7f2091917700 1 -- 172.21.15.152:0/4941741 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f208c0fed80 msgr2=0x7f208c0faec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:30.945 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.939+0000 7f2091917700 1 --2- 172.21.15.152:0/4941741 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f208c0fed80 0x7f208c0faec0 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 crypto rx=0x7f2080004940 tx=0x7f2080009e30 comp rx=0 tx=0).stop 2024-04-02T04:54:30.945 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.939+0000 7f2091917700 1 -- 172.21.15.152:0/4941741 shutdown_connections 2024-04-02T04:54:30.945 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.939+0000 7f2091917700 1 --2- 172.21.15.152:0/4941741 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f208c0fed80 0x7f208c0faec0 unknown :-1 s=CLOSED pgs=175 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:30.945 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.939+0000 7f2091917700 1 --2- 172.21.15.152:0/4941741 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f208c0fe430 0x7f208c0fe840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:30.945 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.939+0000 7f2091917700 1 -- 172.21.15.152:0/4941741 >> 172.21.15.152:0/4941741 conn(0x7f208c0f6bd0 msgr2=0x7f208c0f9000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:54:30.945 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.939+0000 7f2091917700 1 -- 172.21.15.152:0/4941741 shutdown_connections 2024-04-02T04:54:30.945 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.939+0000 7f2091917700 1 -- 172.21.15.152:0/4941741 wait complete. 2024-04-02T04:54:30.945 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.940+0000 7f2091917700 1 Processor -- start 2024-04-02T04:54:30.946 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.940+0000 7f2091917700 1 -- start start 2024-04-02T04:54:30.946 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.940+0000 7f2091917700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f208c0fe430 0x7f208c10d870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:30.946 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.940+0000 7f2091917700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f208c0fed80 0x7f208c10ddb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:30.946 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.940+0000 7f2091917700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f208c10e3b0 con 0x7f208c0fe430 2024-04-02T04:54:30.946 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.940+0000 7f2091917700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f208c10e520 con 0x7f208c0fed80 2024-04-02T04:54:30.946 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.940+0000 7f208affd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f208c0fe430 0x7f208c10d870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:30.946 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.940+0000 7f208affd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f208c0fe430 0x7f208c10d870 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:51474/0 (socket says 172.21.15.152:51474) 2024-04-02T04:54:30.946 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.940+0000 7f208affd700 1 -- 172.21.15.152:0/2624299964 learned_addr learned my addr 172.21.15.152:0/2624299964 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:54:30.946 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.940+0000 7f208a7fc700 1 --2- 172.21.15.152:0/2624299964 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f208c0fed80 0x7f208c10ddb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:30.946 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.940+0000 7f208affd700 1 -- 172.21.15.152:0/2624299964 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f208c0fed80 msgr2=0x7f208c10ddb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:30.946 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.940+0000 7f208affd700 1 --2- 172.21.15.152:0/2624299964 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f208c0fed80 0x7f208c10ddb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:30.946 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.940+0000 7f208affd700 1 -- 172.21.15.152:0/2624299964 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2080005040 con 0x7f208c0fe430 2024-04-02T04:54:30.947 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.940+0000 7f208affd700 1 --2- 172.21.15.152:0/2624299964 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f208c0fe430 0x7f208c10d870 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f208400ec90 tx=0x7f208400b040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:54:30.947 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.941+0000 7f20737fe700 1 -- 172.21.15.152:0/2624299964 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f2084007b30 con 0x7f208c0fe430 2024-04-02T04:54:30.947 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.941+0000 7f20737fe700 1 -- 172.21.15.152:0/2624299964 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f208400f040 con 0x7f208c0fe430 2024-04-02T04:54:30.947 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.941+0000 7f2091917700 1 -- 172.21.15.152:0/2624299964 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f208c112f80 con 0x7f208c0fe430 2024-04-02T04:54:30.947 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.941+0000 7f20737fe700 1 -- 172.21.15.152:0/2624299964 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f208400cc80 con 0x7f208c0fe430 2024-04-02T04:54:30.947 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.941+0000 7f2091917700 1 -- 172.21.15.152:0/2624299964 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f208c113430 con 0x7f208c0fe430 2024-04-02T04:54:30.948 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.942+0000 7f2091917700 1 -- 172.21.15.152:0/2624299964 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f208c04c710 con 0x7f208c0fe430 2024-04-02T04:54:30.948 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.942+0000 7f20737fe700 1 -- 172.21.15.152:0/2624299964 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f2084007c90 con 0x7f208c0fe430 2024-04-02T04:54:30.948 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.943+0000 7f20737fe700 1 --2- 172.21.15.152:0/2624299964 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f20740692a0 0x7f207406b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:30.948 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.943+0000 7f20737fe700 1 -- 172.21.15.152:0/2624299964 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(37..37 src has 1..37) v4 ==== 4338+0+0 (secure 0 0 0) 0x7f20840871a0 con 0x7f208c0fe430 2024-04-02T04:54:30.948 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.943+0000 7f208a7fc700 1 --2- 172.21.15.152:0/2624299964 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f20740692a0 0x7f207406b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:30.948 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.943+0000 7f208a7fc700 1 --2- 172.21.15.152:0/2624299964 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f20740692a0 0x7f207406b750 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f2080009b30 tx=0x7f2080009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:54:30.951 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:30.947+0000 7f20737fe700 1 -- 172.21.15.152:0/2624299964 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f2084057340 con 0x7f208c0fe430 2024-04-02T04:54:31.142 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:31.138+0000 7f2091917700 1 -- 172.21.15.152:0/2624299964 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "osd stat", "format": "json"} v 0) v1 -- 0x7f208c113620 con 0x7f208c0fe430 2024-04-02T04:54:31.142 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:31.139+0000 7f20737fe700 1 -- 172.21.15.152:0/2624299964 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "osd stat", "format": "json"}]=0 v38) v1 ==== 74+0+130 (secure 0 0 0) 0x7f2084056ed0 con 0x7f208c0fe430 2024-04-02T04:54:31.143 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:54:31.144 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:31.141+0000 7f2091917700 1 -- 172.21.15.152:0/2624299964 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f20740692a0 msgr2=0x7f207406b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:31.145 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:31.141+0000 7f2091917700 1 --2- 172.21.15.152:0/2624299964 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f20740692a0 0x7f207406b750 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 crypto rx=0x7f2080009b30 tx=0x7f2080009e30 comp rx=0 tx=0).stop 2024-04-02T04:54:31.145 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:31.141+0000 7f2091917700 1 -- 172.21.15.152:0/2624299964 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f208c0fe430 msgr2=0x7f208c10d870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:31.145 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:31.141+0000 7f2091917700 1 --2- 172.21.15.152:0/2624299964 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f208c0fe430 0x7f208c10d870 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 crypto rx=0x7f208400ec90 tx=0x7f208400b040 comp rx=0 tx=0).stop 2024-04-02T04:54:31.145 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:31.141+0000 7f2091917700 1 -- 172.21.15.152:0/2624299964 shutdown_connections 2024-04-02T04:54:31.145 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:31.141+0000 7f2091917700 1 --2- 172.21.15.152:0/2624299964 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f20740692a0 0x7f207406b750 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:31.145 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:31.141+0000 7f2091917700 1 --2- 172.21.15.152:0/2624299964 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f208c0fe430 0x7f208c10d870 unknown :-1 s=CLOSED pgs=176 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:31.145 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:31.141+0000 7f2091917700 1 --2- 172.21.15.152:0/2624299964 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f208c0fed80 0x7f208c10ddb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:31.145 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:31.141+0000 7f2091917700 1 -- 172.21.15.152:0/2624299964 >> 172.21.15.152:0/2624299964 conn(0x7f208c0f6bd0 msgr2=0x7f208c0f8180 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:54:31.145 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:31.141+0000 7f2091917700 1 -- 172.21.15.152:0/2624299964 shutdown_connections 2024-04-02T04:54:31.145 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:31.141+0000 7f2091917700 1 -- 172.21.15.152:0/2624299964 wait complete. 2024-04-02T04:54:31.279 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:31 smithi152 ceph-mon[29073]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:31.279 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:31 smithi152 ceph-mon[29073]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-02T04:54:31.279 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:31 smithi152 ceph-mon[29073]: osdmap e37: 6 total, 5 up, 6 in 2024-04-02T04:54:31.279 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:31 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:54:31.280 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:31 smithi152 ceph-mon[29073]: from='osd.5 [v2:172.21.15.154:6816/1178927498,v1:172.21.15.154:6817/1178927498]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-04-02T04:54:31.280 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:31 smithi152 ceph-mon[29073]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-04-02T04:54:31.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:31 smithi154 ceph-mon[35729]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:31.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:31 smithi154 ceph-mon[35729]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-02T04:54:31.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:31 smithi154 ceph-mon[35729]: osdmap e37: 6 total, 5 up, 6 in 2024-04-02T04:54:31.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:31 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:54:31.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:31 smithi154 ceph-mon[35729]: from='osd.5 [v2:172.21.15.154:6816/1178927498,v1:172.21.15.154:6817/1178927498]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-04-02T04:54:31.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:31 smithi154 ceph-mon[35729]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]: dispatch 2024-04-02T04:54:31.484 INFO:journalctl@ceph.osd.5.smithi154.stdout:Apr 02 04:54:31 smithi154 ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-osd-5[54772]: 2024-04-02T04:54:31.120+0000 7f10285a2700 -1 osd.5 0 waiting for initial osdmap 2024-04-02T04:54:31.484 INFO:journalctl@ceph.osd.5.smithi154.stdout:Apr 02 04:54:31 smithi154 ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-osd-5[54772]: 2024-04-02T04:54:31.130+0000 7f102239b700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-02T04:54:31.840 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":38,"num_osds":6,"num_up_osds":5,"osd_up_since":1712033646,"num_in_osds":6,"osd_in_since":1712033653,"num_remapped_pgs":0} 2024-04-02T04:54:32.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:32 smithi154 ceph-mon[35729]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-04-02T04:54:32.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:32 smithi154 ceph-mon[35729]: osdmap e38: 6 total, 5 up, 6 in 2024-04-02T04:54:32.234 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:54:32.234 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:32 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:54:32.234 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:32 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/2624299964' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-02T04:54:32.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:32 smithi152 ceph-mon[29073]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi154", "root=default"]}]': finished 2024-04-02T04:54:32.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:32 smithi152 ceph-mon[29073]: osdmap e38: 6 total, 5 up, 6 in 2024-04-02T04:54:32.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:32 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:54:32.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:32 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:54:32.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:32 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/2624299964' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-02T04:54:32.841 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph osd stat -f json 2024-04-02T04:54:33.102 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:54:33.299 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:33 smithi154 ceph-mon[35729]: purged_snaps scrub starts 2024-04-02T04:54:33.299 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:33 smithi154 ceph-mon[35729]: purged_snaps scrub ok 2024-04-02T04:54:33.299 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:33 smithi154 ceph-mon[35729]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:33.299 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:33 smithi154 ceph-mon[35729]: osd.5 [v2:172.21.15.154:6816/1178927498,v1:172.21.15.154:6817/1178927498] boot 2024-04-02T04:54:33.299 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:33 smithi154 ceph-mon[35729]: osdmap e39: 6 total, 6 up, 6 in 2024-04-02T04:54:33.299 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:33 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:54:33.392 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:33 smithi152 ceph-mon[29073]: purged_snaps scrub starts 2024-04-02T04:54:33.392 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:33 smithi152 ceph-mon[29073]: purged_snaps scrub ok 2024-04-02T04:54:33.392 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:33 smithi152 ceph-mon[29073]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-04-02T04:54:33.392 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:33 smithi152 ceph-mon[29073]: osd.5 [v2:172.21.15.154:6816/1178927498,v1:172.21.15.154:6817/1178927498] boot 2024-04-02T04:54:33.392 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:33 smithi152 ceph-mon[29073]: osdmap e39: 6 total, 6 up, 6 in 2024-04-02T04:54:33.392 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:33 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-02T04:54:34.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:34 smithi154 ceph-mon[35729]: osdmap e40: 6 total, 6 up, 6 in 2024-04-02T04:54:34.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:34 smithi154 ceph-mon[35729]: Detected new or changed devices on smithi154 2024-04-02T04:54:34.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:34 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:34.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:34 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:34.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:34 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:54:34.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:34 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:54:34.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:34 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:54:34.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:34 smithi154 ceph-mon[35729]: Adjusting osd_memory_target on smithi154 to 5312M 2024-04-02T04:54:34.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:34 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:34.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:34 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:54:34.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:34 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:54:34.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:34 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:34.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:34 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:54:34.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:34 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:54:34.485 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:34 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:54:34.485 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:34 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:34.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:34 smithi152 ceph-mon[29073]: osdmap e40: 6 total, 6 up, 6 in 2024-04-02T04:54:34.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:34 smithi152 ceph-mon[29073]: Detected new or changed devices on smithi154 2024-04-02T04:54:34.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:34 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:34.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:34 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:34.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:34 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:54:34.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:34 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:54:34.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:34 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-02T04:54:34.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:34 smithi152 ceph-mon[29073]: Adjusting osd_memory_target on smithi154 to 5312M 2024-04-02T04:54:34.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:34 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:34.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:34 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:54:34.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:34 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:54:34.508 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:34 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:34.508 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:34 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:54:34.508 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:34 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:54:34.508 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:34 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:54:34.508 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:34 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:54:34.885 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.880+0000 7fe5f65a6700 1 -- 172.21.15.152:0/1496780845 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe5f00fe450 msgr2=0x7fe5f00fe860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:34.885 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.880+0000 7fe5f65a6700 1 --2- 172.21.15.152:0/1496780845 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe5f00fe450 0x7fe5f00fe860 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 crypto rx=0x7fe5e0004940 tx=0x7fe5e0009e30 comp rx=0 tx=0).stop 2024-04-02T04:54:34.885 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.880+0000 7fe5f65a6700 1 -- 172.21.15.152:0/1496780845 shutdown_connections 2024-04-02T04:54:34.886 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.880+0000 7fe5f65a6700 1 --2- 172.21.15.152:0/1496780845 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe5f00feda0 0x7fe5f00faee0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:34.886 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.880+0000 7fe5f65a6700 1 --2- 172.21.15.152:0/1496780845 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe5f00fe450 0x7fe5f00fe860 unknown :-1 s=CLOSED pgs=177 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:34.886 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.880+0000 7fe5f65a6700 1 -- 172.21.15.152:0/1496780845 >> 172.21.15.152:0/1496780845 conn(0x7fe5f00f6c10 msgr2=0x7fe5f00f9020 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:54:34.886 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.880+0000 7fe5f65a6700 1 -- 172.21.15.152:0/1496780845 shutdown_connections 2024-04-02T04:54:34.886 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.880+0000 7fe5f65a6700 1 -- 172.21.15.152:0/1496780845 wait complete. 2024-04-02T04:54:34.886 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.880+0000 7fe5f65a6700 1 Processor -- start 2024-04-02T04:54:34.886 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.881+0000 7fe5f65a6700 1 -- start start 2024-04-02T04:54:34.886 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.881+0000 7fe5f65a6700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe5f00fe450 0x7fe5f010d930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:34.886 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.881+0000 7fe5f65a6700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe5f00feda0 0x7fe5f010de70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:34.886 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.881+0000 7fe5f65a6700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe5f010e470 con 0x7fe5f00feda0 2024-04-02T04:54:34.886 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.881+0000 7fe5f65a6700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe5f010e5e0 con 0x7fe5f00fe450 2024-04-02T04:54:34.886 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.881+0000 7fe5e7fff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe5f00feda0 0x7fe5f010de70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:34.886 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.881+0000 7fe5e7fff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe5f00feda0 0x7fe5f010de70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:51484/0 (socket says 172.21.15.152:51484) 2024-04-02T04:54:34.887 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.881+0000 7fe5e7fff700 1 -- 172.21.15.152:0/1018830430 learned_addr learned my addr 172.21.15.152:0/1018830430 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:54:34.887 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.881+0000 7fe5effff700 1 --2- 172.21.15.152:0/1018830430 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe5f00fe450 0x7fe5f010d930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:34.887 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.881+0000 7fe5e7fff700 1 -- 172.21.15.152:0/1018830430 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe5f00fe450 msgr2=0x7fe5f010d930 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:34.887 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.881+0000 7fe5e7fff700 1 --2- 172.21.15.152:0/1018830430 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe5f00fe450 0x7fe5f010d930 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:34.887 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.881+0000 7fe5e7fff700 1 -- 172.21.15.152:0/1018830430 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe5e0005040 con 0x7fe5f00feda0 2024-04-02T04:54:34.887 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.881+0000 7fe5effff700 1 --2- 172.21.15.152:0/1018830430 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe5f00fe450 0x7fe5f010d930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-02T04:54:34.887 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.881+0000 7fe5e7fff700 1 --2- 172.21.15.152:0/1018830430 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe5f00feda0 0x7fe5f010de70 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7fe5d800c8c0 tx=0x7fe5d80044c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:54:34.887 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.881+0000 7fe5ed7fa700 1 -- 172.21.15.152:0/1018830430 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fe5d800a980 con 0x7fe5f00feda0 2024-04-02T04:54:34.887 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.881+0000 7fe5ed7fa700 1 -- 172.21.15.152:0/1018830430 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fe5d800ce60 con 0x7fe5f00feda0 2024-04-02T04:54:34.887 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.881+0000 7fe5f65a6700 1 -- 172.21.15.152:0/1018830430 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe5f0113040 con 0x7fe5f00feda0 2024-04-02T04:54:34.887 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.881+0000 7fe5ed7fa700 1 -- 172.21.15.152:0/1018830430 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fe5d80115e0 con 0x7fe5f00feda0 2024-04-02T04:54:34.887 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.881+0000 7fe5f65a6700 1 -- 172.21.15.152:0/1018830430 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe5f01135d0 con 0x7fe5f00feda0 2024-04-02T04:54:34.888 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.882+0000 7fe5f65a6700 1 -- 172.21.15.152:0/1018830430 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe5f004c710 con 0x7fe5f00feda0 2024-04-02T04:54:34.888 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.883+0000 7fe5ed7fa700 1 -- 172.21.15.152:0/1018830430 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7fe5d8012400 con 0x7fe5f00feda0 2024-04-02T04:54:34.888 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.883+0000 7fe5ed7fa700 1 --2- 172.21.15.152:0/1018830430 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fe5dc0692a0 0x7fe5dc06b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:34.888 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.883+0000 7fe5ed7fa700 1 -- 172.21.15.152:0/1018830430 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7fe5d8086e40 con 0x7fe5f00feda0 2024-04-02T04:54:34.889 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.883+0000 7fe5effff700 1 --2- 172.21.15.152:0/1018830430 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fe5dc0692a0 0x7fe5dc06b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:34.889 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.884+0000 7fe5effff700 1 --2- 172.21.15.152:0/1018830430 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fe5dc0692a0 0x7fe5dc06b750 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7fe5e000b3b0 tx=0x7fe5e0009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:54:34.891 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:34.888+0000 7fe5ed7fa700 1 -- 172.21.15.152:0/1018830430 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fe5d8056fe0 con 0x7fe5f00feda0 2024-04-02T04:54:35.075 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:35.072+0000 7fe5f65a6700 1 -- 172.21.15.152:0/1018830430 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "osd stat", "format": "json"} v 0) v1 -- 0x7fe5f0105900 con 0x7fe5f00feda0 2024-04-02T04:54:35.076 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:35.072+0000 7fe5ed7fa700 1 -- 172.21.15.152:0/1018830430 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "osd stat", "format": "json"}]=0 v40) v1 ==== 74+0+130 (secure 0 0 0) 0x7fe5d80126b0 con 0x7fe5f00feda0 2024-04-02T04:54:35.076 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:54:35.078 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:35.074+0000 7fe5f65a6700 1 -- 172.21.15.152:0/1018830430 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fe5dc0692a0 msgr2=0x7fe5dc06b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:35.078 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:35.074+0000 7fe5f65a6700 1 --2- 172.21.15.152:0/1018830430 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fe5dc0692a0 0x7fe5dc06b750 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 crypto rx=0x7fe5e000b3b0 tx=0x7fe5e0009e30 comp rx=0 tx=0).stop 2024-04-02T04:54:35.078 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:35.074+0000 7fe5f65a6700 1 -- 172.21.15.152:0/1018830430 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe5f00feda0 msgr2=0x7fe5f010de70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:35.078 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:35.074+0000 7fe5f65a6700 1 --2- 172.21.15.152:0/1018830430 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe5f00feda0 0x7fe5f010de70 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 crypto rx=0x7fe5d800c8c0 tx=0x7fe5d80044c0 comp rx=0 tx=0).stop 2024-04-02T04:54:35.078 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:35.074+0000 7fe5f65a6700 1 -- 172.21.15.152:0/1018830430 shutdown_connections 2024-04-02T04:54:35.078 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:35.074+0000 7fe5f65a6700 1 --2- 172.21.15.152:0/1018830430 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fe5dc0692a0 0x7fe5dc06b750 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:35.078 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:35.074+0000 7fe5f65a6700 1 --2- 172.21.15.152:0/1018830430 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe5f00fe450 0x7fe5f010d930 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:35.078 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:35.074+0000 7fe5f65a6700 1 --2- 172.21.15.152:0/1018830430 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe5f00feda0 0x7fe5f010de70 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:35.079 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:35.074+0000 7fe5f65a6700 1 -- 172.21.15.152:0/1018830430 >> 172.21.15.152:0/1018830430 conn(0x7fe5f00f6c10 msgr2=0x7fe5f00f82d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:54:35.079 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:35.074+0000 7fe5f65a6700 1 -- 172.21.15.152:0/1018830430 shutdown_connections 2024-04-02T04:54:35.079 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:35.074+0000 7fe5f65a6700 1 -- 172.21.15.152:0/1018830430 wait complete. 2024-04-02T04:54:35.746 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:35 smithi152 ceph-mon[29073]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:54:35.746 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:35 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/1018830430' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-02T04:54:35.771 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":40,"num_osds":6,"num_up_osds":6,"osd_up_since":1712033672,"num_in_osds":6,"osd_in_since":1712033653,"num_remapped_pgs":0} 2024-04-02T04:54:35.772 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-02T04:54:35.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:35 smithi154 ceph-mon[35729]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:54:35.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:35 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/1018830430' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-02T04:54:36.025 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:54:37.689 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.683+0000 7f613c34d700 1 -- 172.21.15.152:0/137705123 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61340fe820 msgr2=0x7f6134106e60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:37.690 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.683+0000 7f613c34d700 1 --2- 172.21.15.152:0/137705123 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61340fe820 0x7f6134106e60 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 crypto rx=0x7f6124004940 tx=0x7f6124009e30 comp rx=0 tx=0).stop 2024-04-02T04:54:37.690 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.683+0000 7f613c34d700 1 -- 172.21.15.152:0/137705123 shutdown_connections 2024-04-02T04:54:37.690 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.683+0000 7f613c34d700 1 --2- 172.21.15.152:0/137705123 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61340fe820 0x7f6134106e60 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:37.690 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.683+0000 7f613c34d700 1 --2- 172.21.15.152:0/137705123 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f61340fde80 0x7f61340fe250 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:37.690 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.683+0000 7f613c34d700 1 -- 172.21.15.152:0/137705123 >> 172.21.15.152:0/137705123 conn(0x7f61340f78b0 msgr2=0x7f61340f9cc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:54:37.690 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.683+0000 7f613c34d700 1 -- 172.21.15.152:0/137705123 shutdown_connections 2024-04-02T04:54:37.690 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.683+0000 7f613c34d700 1 -- 172.21.15.152:0/137705123 wait complete. 2024-04-02T04:54:37.690 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.683+0000 7f613c34d700 1 Processor -- start 2024-04-02T04:54:37.690 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.684+0000 7f613c34d700 1 -- start start 2024-04-02T04:54:37.690 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.684+0000 7f613c34d700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f61340fde80 0x7f6134113880 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:37.690 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.684+0000 7f613c34d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61340fe820 0x7f6134113dc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:37.690 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.684+0000 7f613c34d700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6134114410 con 0x7f61340fe820 2024-04-02T04:54:37.690 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.684+0000 7f613c34d700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f613410da10 con 0x7f61340fde80 2024-04-02T04:54:37.691 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.684+0000 7f61398e8700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61340fe820 0x7f6134113dc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:37.691 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.684+0000 7f61398e8700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61340fe820 0x7f6134113dc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:51504/0 (socket says 172.21.15.152:51504) 2024-04-02T04:54:37.691 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.684+0000 7f61398e8700 1 -- 172.21.15.152:0/20299082 learned_addr learned my addr 172.21.15.152:0/20299082 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:54:37.691 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.684+0000 7f613a0e9700 1 --2- 172.21.15.152:0/20299082 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f61340fde80 0x7f6134113880 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:37.691 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.684+0000 7f61398e8700 1 -- 172.21.15.152:0/20299082 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f61340fde80 msgr2=0x7f6134113880 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:37.691 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.684+0000 7f61398e8700 1 --2- 172.21.15.152:0/20299082 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f61340fde80 0x7f6134113880 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:37.691 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.684+0000 7f61398e8700 1 -- 172.21.15.152:0/20299082 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6124005040 con 0x7f61340fe820 2024-04-02T04:54:37.691 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.684+0000 7f613a0e9700 1 --2- 172.21.15.152:0/20299082 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f61340fde80 0x7f6134113880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-02T04:54:37.691 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.684+0000 7f61398e8700 1 --2- 172.21.15.152:0/20299082 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61340fe820 0x7f6134113dc0 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f6124004890 tx=0x7f6124009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:54:37.691 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.684+0000 7f612affd700 1 -- 172.21.15.152:0/20299082 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f612400e040 con 0x7f61340fe820 2024-04-02T04:54:37.691 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.685+0000 7f612affd700 1 -- 172.21.15.152:0/20299082 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f6124009af0 con 0x7f61340fe820 2024-04-02T04:54:37.691 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.685+0000 7f613c34d700 1 -- 172.21.15.152:0/20299082 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f613410dcf0 con 0x7f61340fe820 2024-04-02T04:54:37.691 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.685+0000 7f612affd700 1 -- 172.21.15.152:0/20299082 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f612401c630 con 0x7f61340fe820 2024-04-02T04:54:37.691 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.685+0000 7f613c34d700 1 -- 172.21.15.152:0/20299082 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f613410e230 con 0x7f61340fe820 2024-04-02T04:54:37.691 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.686+0000 7f613c34d700 1 -- 172.21.15.152:0/20299082 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6134066b10 con 0x7f61340fe820 2024-04-02T04:54:37.692 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.686+0000 7f612affd700 1 -- 172.21.15.152:0/20299082 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f612400bbc0 con 0x7f61340fe820 2024-04-02T04:54:37.692 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.686+0000 7f612affd700 1 --2- 172.21.15.152:0/20299082 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f61200692a0 0x7f612006b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:37.692 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.686+0000 7f612affd700 1 -- 172.21.15.152:0/20299082 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7f6124088050 con 0x7f61340fe820 2024-04-02T04:54:37.692 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.686+0000 7f613a0e9700 1 --2- 172.21.15.152:0/20299082 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f61200692a0 0x7f612006b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:37.692 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.687+0000 7f613a0e9700 1 --2- 172.21.15.152:0/20299082 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f61200692a0 0x7f612006b750 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f6130007fd0 tx=0x7f6130009f60 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:54:37.694 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.691+0000 7f612affd700 1 -- 172.21.15.152:0/20299082 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f61240581f0 con 0x7f61340fe820 2024-04-02T04:54:37.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:37 smithi152 ceph-mon[29073]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:54:37.881 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.878+0000 7f613c34d700 1 -- 172.21.15.152:0/20299082 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f613404c710 con 0x7f61340fe820 2024-04-02T04:54:37.882 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.878+0000 7f612affd700 1 -- 172.21.15.152:0/20299082 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v40) v1 ==== 74+0+11112 (secure 0 0 0) 0x7f6124057d80 con 0x7f61340fe820 2024-04-02T04:54:37.882 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:54:37.883 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":40,"fsid":"2e0055d0-f0ac-11ee-b647-cb9ed24678a4","created":"2024-04-02T04:48:59.746166+0000","modified":"2024-04-02T04:54:33.124544+0000","last_up_change":"2024-04-02T04:54:32.119050+0000","last_in_change":"2024-04-02T04:54:13.064706+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":14,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":6,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-02T04:53:20.274250+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"3f25c359-f643-4dfb-ba7b-30bb1a2dfcb2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":718734380},{"type":"v1","addr":"172.21.15.152:6803","nonce":718734380}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":718734380},{"type":"v1","addr":"172.21.15.152:6805","nonce":718734380}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":718734380},{"type":"v1","addr":"172.21.15.152:6809","nonce":718734380}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":718734380},{"type":"v1","addr":"172.21.15.152:6807","nonce":718734380}]},"public_addr":"172.21.15.152:6803/718734380","cluster_addr":"172.21.15.152:6805/718734380","heartbeat_back_addr":"172.21.15.152:6809/718734380","heartbeat_front_addr":"172.21.15.152:6807/718734380","state":["exists","up"]},{"osd":1,"uuid":"d4555f9e-fc0e-4266-ae65-3c197d3fa02c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":29,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":782616861},{"type":"v1","addr":"172.21.15.152:6811","nonce":782616861}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":782616861},{"type":"v1","addr":"172.21.15.152:6813","nonce":782616861}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":782616861},{"type":"v1","addr":"172.21.15.152:6817","nonce":782616861}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":782616861},{"type":"v1","addr":"172.21.15.152:6815","nonce":782616861}]},"public_addr":"172.21.15.152:6811/782616861","cluster_addr":"172.21.15.152:6813/782616861","heartbeat_back_addr":"172.21.15.152:6817/782616861","heartbeat_front_addr":"172.21.15.152:6815/782616861","state":["exists","up"]},{"osd":2,"uuid":"caa0a912-a9d5-45e7-afe2-51beaf0f1742","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":1566571469},{"type":"v1","addr":"172.21.15.152:6819","nonce":1566571469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":1566571469},{"type":"v1","addr":"172.21.15.152:6821","nonce":1566571469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":1566571469},{"type":"v1","addr":"172.21.15.152:6825","nonce":1566571469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":1566571469},{"type":"v1","addr":"172.21.15.152:6823","nonce":1566571469}]},"public_addr":"172.21.15.152:6819/1566571469","cluster_addr":"172.21.15.152:6821/1566571469","heartbeat_back_addr":"172.21.15.152:6825/1566571469","heartbeat_front_addr":"172.21.15.152:6823/1566571469","state":["exists","up"]},{"osd":3,"uuid":"84a2d337-6ac8-41e4-a922-a6e94c1e9067","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6800","nonce":3366219775},{"type":"v1","addr":"172.21.15.154:6801","nonce":3366219775}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6802","nonce":3366219775},{"type":"v1","addr":"172.21.15.154:6803","nonce":3366219775}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6806","nonce":3366219775},{"type":"v1","addr":"172.21.15.154:6807","nonce":3366219775}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6804","nonce":3366219775},{"type":"v1","addr":"172.21.15.154:6805","nonce":3366219775}]},"public_addr":"172.21.15.154:6801/3366219775","cluster_addr":"172.21.15.154:6803/3366219775","heartbeat_back_addr":"172.21.15.154:6807/3366219775","heartbeat_front_addr":"172.21.15.154:6805/3366219775","state":["exists","up"]},{"osd":4,"uuid":"d547308c-898c-4165-9762-c8fd4170dc39","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":34,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6808","nonce":220818867},{"type":"v1","addr":"172.21.15.154:6809","nonce":220818867}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6810","nonce":220818867},{"type":"v1","addr":"172.21.15.154:6811","nonce":220818867}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6814","nonce":220818867},{"type":"v1","addr":"172.21.15.154:6815","nonce":220818867}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6812","nonce":220818867},{"type":"v1","addr":"172.21.15.154:6813","nonce":220818867}]},"public_addr":"172.21.15.154:6809/220818867","cluster_addr":"172.21.15.154:6811/220818867","heartbeat_back_addr":"172.21.15.154:6815/220818867","heartbeat_front_addr":"172.21.15.154:6813/220818867","state":["exists","up"]},{"osd":5,"uuid":"c3d3760a-08a7-44f2-8288-ccc5d08df057","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6816","nonce":1178927498},{"type":"v1","addr":"172.21.15.154:6817","nonce":1178927498}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6818","nonce":1178927498},{"type":"v1","addr":"172.21.15.154:6819","nonce":1178927498}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6822","nonce":1178927498},{"type":"v1","addr":"172.21.15.154:6823","nonce":1178927498}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6820","nonce":1178927498},{"type":"v1","addr":"172.21.15.154:6821","nonce":1178927498}]},"public_addr":"172.21.15.154:6817/1178927498","cluster_addr":"172.21.15.154:6819/1178927498","heartbeat_back_addr":"172.21.15.154:6823/1178927498","heartbeat_front_addr":"172.21.15.154:6821/1178927498","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:52:20.472492+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:52:49.979314+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:53:17.640952+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:53:40.392841+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:54:05.761710+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:54:30.235521+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.152:0/3027168373":"2024-04-03T04:51:09.949838+0000","172.21.15.152:0/2268166004":"2024-04-03T04:51:09.949838+0000","172.21.15.152:6800/2497421829":"2024-04-03T04:51:09.949838+0000","172.21.15.152:6801/2497421829":"2024-04-03T04:51:09.949838+0000","172.21.15.152:0/4188679877":"2024-04-03T04:50:01.902008+0000","172.21.15.152:0/1860988983":"2024-04-03T04:50:01.902008+0000","172.21.15.152:0/1943568671":"2024-04-03T04:49:30.819772+0000","172.21.15.152:0/2491428539":"2024-04-03T04:51:09.949838+0000","172.21.15.152:6801/3544077480":"2024-04-03T04:50:01.902008+0000","172.21.15.152:0/3476288520":"2024-04-03T04:49:30.819772+0000","172.21.15.152:6800/2913738627":"2024-04-03T04:49:30.819772+0000","172.21.15.152:6800/3544077480":"2024-04-03T04:50:01.902008+0000","172.21.15.152:0/3023512755":"2024-04-03T04:49:30.819772+0000","172.21.15.152:6801/2913738627":"2024-04-03T04:49:30.819772+0000","172.21.15.152:0/2947158817":"2024-04-03T04:50:01.902008+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-02T04:54:37.884 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.880+0000 7f613c34d700 1 -- 172.21.15.152:0/20299082 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f61200692a0 msgr2=0x7f612006b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:37.884 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.880+0000 7f613c34d700 1 --2- 172.21.15.152:0/20299082 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f61200692a0 0x7f612006b750 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 crypto rx=0x7f6130007fd0 tx=0x7f6130009f60 comp rx=0 tx=0).stop 2024-04-02T04:54:37.885 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.880+0000 7f613c34d700 1 -- 172.21.15.152:0/20299082 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61340fe820 msgr2=0x7f6134113dc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:37.885 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.880+0000 7f613c34d700 1 --2- 172.21.15.152:0/20299082 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61340fe820 0x7f6134113dc0 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 crypto rx=0x7f6124004890 tx=0x7f6124009e30 comp rx=0 tx=0).stop 2024-04-02T04:54:37.885 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.880+0000 7f613c34d700 1 -- 172.21.15.152:0/20299082 shutdown_connections 2024-04-02T04:54:37.885 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.880+0000 7f613c34d700 1 --2- 172.21.15.152:0/20299082 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f61200692a0 0x7f612006b750 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:37.885 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.880+0000 7f613c34d700 1 --2- 172.21.15.152:0/20299082 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f61340fde80 0x7f6134113880 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:37.885 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.881+0000 7f613c34d700 1 --2- 172.21.15.152:0/20299082 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61340fe820 0x7f6134113dc0 unknown :-1 s=CLOSED pgs=180 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:37.885 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.881+0000 7f613c34d700 1 -- 172.21.15.152:0/20299082 >> 172.21.15.152:0/20299082 conn(0x7f61340f78b0 msgr2=0x7f61340f8d10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:54:37.885 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.881+0000 7f613c34d700 1 -- 172.21.15.152:0/20299082 shutdown_connections 2024-04-02T04:54:37.885 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:37.881+0000 7f613c34d700 1 -- 172.21.15.152:0/20299082 wait complete. 2024-04-02T04:54:37.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:37 smithi154 ceph-mon[35729]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:54:38.501 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-02T04:53:20.274250+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-02T04:54:38.501 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph osd pool get .mgr pg_num 2024-04-02T04:54:38.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:38 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/20299082' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T04:54:38.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:38 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/20299082' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T04:54:38.758 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:54:39.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:39 smithi152 ceph-mon[29073]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:54:39.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:39 smithi154 ceph-mon[35729]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:54:40.338 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.332+0000 7f0803254700 1 -- 172.21.15.152:0/2997050622 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f07fc0fde80 msgr2=0x7f07fc0fe250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:40.338 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.332+0000 7f0803254700 1 --2- 172.21.15.152:0/2997050622 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f07fc0fde80 0x7f07fc0fe250 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 crypto rx=0x7f07f0004940 tx=0x7f07f0009e30 comp rx=0 tx=0).stop 2024-04-02T04:54:40.338 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.333+0000 7f0803254700 1 -- 172.21.15.152:0/2997050622 shutdown_connections 2024-04-02T04:54:40.338 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.333+0000 7f0803254700 1 --2- 172.21.15.152:0/2997050622 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f07fc0fe820 0x7f07fc106e60 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:40.338 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.333+0000 7f0803254700 1 --2- 172.21.15.152:0/2997050622 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f07fc0fde80 0x7f07fc0fe250 unknown :-1 s=CLOSED pgs=181 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:40.338 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.333+0000 7f0803254700 1 -- 172.21.15.152:0/2997050622 >> 172.21.15.152:0/2997050622 conn(0x7f07fc0f7870 msgr2=0x7f07fc0f9c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:54:40.339 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.333+0000 7f0803254700 1 -- 172.21.15.152:0/2997050622 shutdown_connections 2024-04-02T04:54:40.339 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.333+0000 7f0803254700 1 -- 172.21.15.152:0/2997050622 wait complete. 2024-04-02T04:54:40.339 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.333+0000 7f0803254700 1 Processor -- start 2024-04-02T04:54:40.339 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.333+0000 7f0803254700 1 -- start start 2024-04-02T04:54:40.339 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.333+0000 7f0803254700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f07fc0fde80 0x7f07fc113910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:40.339 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.333+0000 7f0803254700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f07fc0fe820 0x7f07fc113e50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:40.339 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.334+0000 7f0803254700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f07fc1144a0 con 0x7f07fc0fe820 2024-04-02T04:54:40.339 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.334+0000 7f0803254700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f07fc10daa0 con 0x7f07fc0fde80 2024-04-02T04:54:40.339 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.334+0000 7f07fbfff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f07fc0fe820 0x7f07fc113e50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:40.339 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.334+0000 7f07fbfff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f07fc0fe820 0x7f07fc113e50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:51524/0 (socket says 172.21.15.152:51524) 2024-04-02T04:54:40.339 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.334+0000 7f07fbfff700 1 -- 172.21.15.152:0/1249615336 learned_addr learned my addr 172.21.15.152:0/1249615336 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:54:40.339 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.334+0000 7f0800ff0700 1 --2- 172.21.15.152:0/1249615336 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f07fc0fde80 0x7f07fc113910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:40.339 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.334+0000 7f07fbfff700 1 -- 172.21.15.152:0/1249615336 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f07fc0fde80 msgr2=0x7f07fc113910 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:40.340 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.334+0000 7f07fbfff700 1 --2- 172.21.15.152:0/1249615336 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f07fc0fde80 0x7f07fc113910 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:40.340 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.334+0000 7f07fbfff700 1 -- 172.21.15.152:0/1249615336 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f07f0005040 con 0x7f07fc0fe820 2024-04-02T04:54:40.340 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.334+0000 7f07fbfff700 1 --2- 172.21.15.152:0/1249615336 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f07fc0fe820 0x7f07fc113e50 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f07e8002a50 tx=0x7f07e80046b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:54:40.340 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.334+0000 7f07f97fa700 1 -- 172.21.15.152:0/1249615336 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f07e8007cc0 con 0x7f07fc0fe820 2024-04-02T04:54:40.340 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.334+0000 7f0803254700 1 -- 172.21.15.152:0/1249615336 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f07fc10dd80 con 0x7f07fc0fe820 2024-04-02T04:54:40.340 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.334+0000 7f07f97fa700 1 -- 172.21.15.152:0/1249615336 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f07e800f040 con 0x7f07fc0fe820 2024-04-02T04:54:40.340 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.334+0000 7f07f97fa700 1 -- 172.21.15.152:0/1249615336 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f07e800da30 con 0x7f07fc0fe820 2024-04-02T04:54:40.340 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.334+0000 7f0803254700 1 -- 172.21.15.152:0/1249615336 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f07fc10e2c0 con 0x7f07fc0fe820 2024-04-02T04:54:40.341 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.335+0000 7f0803254700 1 -- 172.21.15.152:0/1249615336 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f07fc066af0 con 0x7f07fc0fe820 2024-04-02T04:54:40.341 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.336+0000 7f07f97fa700 1 -- 172.21.15.152:0/1249615336 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f07e800db90 con 0x7f07fc0fe820 2024-04-02T04:54:40.341 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.336+0000 7f07f97fa700 1 --2- 172.21.15.152:0/1249615336 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f07ec0692a0 0x7f07ec06b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:40.341 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.336+0000 7f07f97fa700 1 -- 172.21.15.152:0/1249615336 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7f07e8086c00 con 0x7f07fc0fe820 2024-04-02T04:54:40.341 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.336+0000 7f0800ff0700 1 --2- 172.21.15.152:0/1249615336 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f07ec0692a0 0x7f07ec06b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:40.341 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.337+0000 7f0800ff0700 1 --2- 172.21.15.152:0/1249615336 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f07ec0692a0 0x7f07ec06b750 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f07f0004f90 tx=0x7f07f0009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:54:40.344 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.340+0000 7f07f97fa700 1 -- 172.21.15.152:0/1249615336 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f07e808c020 con 0x7f07fc0fe820 2024-04-02T04:54:40.530 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.526+0000 7f0803254700 1 -- 172.21.15.152:0/1249615336 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f07fc04c710 con 0x7f07fc0fe820 2024-04-02T04:54:40.531 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.527+0000 7f07f97fa700 1 -- 172.21.15.152:0/1249615336 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v40) v1 ==== 93+0+10 (secure 0 0 0) 0x7f07e8056c80 con 0x7f07fc0fe820 2024-04-02T04:54:40.531 INFO:teuthology.orchestra.run.smithi152.stdout:pg_num: 1 2024-04-02T04:54:40.532 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.529+0000 7f0803254700 1 -- 172.21.15.152:0/1249615336 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f07ec0692a0 msgr2=0x7f07ec06b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:40.532 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.529+0000 7f0803254700 1 --2- 172.21.15.152:0/1249615336 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f07ec0692a0 0x7f07ec06b750 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 crypto rx=0x7f07f0004f90 tx=0x7f07f0009e30 comp rx=0 tx=0).stop 2024-04-02T04:54:40.533 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.529+0000 7f0803254700 1 -- 172.21.15.152:0/1249615336 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f07fc0fe820 msgr2=0x7f07fc113e50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:40.533 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.529+0000 7f0803254700 1 --2- 172.21.15.152:0/1249615336 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f07fc0fe820 0x7f07fc113e50 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 crypto rx=0x7f07e8002a50 tx=0x7f07e80046b0 comp rx=0 tx=0).stop 2024-04-02T04:54:40.533 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.529+0000 7f0803254700 1 -- 172.21.15.152:0/1249615336 shutdown_connections 2024-04-02T04:54:40.533 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.529+0000 7f0803254700 1 --2- 172.21.15.152:0/1249615336 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f07ec0692a0 0x7f07ec06b750 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:40.533 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.529+0000 7f0803254700 1 --2- 172.21.15.152:0/1249615336 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f07fc0fde80 0x7f07fc113910 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:40.533 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.529+0000 7f0803254700 1 --2- 172.21.15.152:0/1249615336 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f07fc0fe820 0x7f07fc113e50 unknown :-1 s=CLOSED pgs=182 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:40.533 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.529+0000 7f0803254700 1 -- 172.21.15.152:0/1249615336 >> 172.21.15.152:0/1249615336 conn(0x7f07fc0f7870 msgr2=0x7f07fc0f8e00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:54:40.533 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.529+0000 7f0803254700 1 -- 172.21.15.152:0/1249615336 shutdown_connections 2024-04-02T04:54:40.533 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:40.529+0000 7f0803254700 1 -- 172.21.15.152:0/1249615336 wait complete. 2024-04-02T04:54:40.847 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:40 smithi152 ceph-mon[29073]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:54:40.848 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:40 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:54:40.903 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:40 smithi154 ceph-mon[35729]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:54:40.903 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:40 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:54:41.258 INFO:tasks.cephadm:Setting up client nodes... 2024-04-02T04:54:41.258 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-02T04:54:41.515 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:54:41.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:41 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/1249615336' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-02T04:54:41.828 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:41 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/1249615336' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-02T04:54:42.555 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:42 smithi152 ceph-mon[29073]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:54:42.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:42 smithi154 ceph-mon[35729]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:54:43.051 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.045+0000 7f94c9fa1700 1 -- 172.21.15.152:0/2360444340 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f94c40fffb0 msgr2=0x7f94c4102390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:43.051 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.045+0000 7f94c9fa1700 1 --2- 172.21.15.152:0/2360444340 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f94c40fffb0 0x7f94c4102390 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 crypto rx=0x7f94ac004940 tx=0x7f94ac009e30 comp rx=0 tx=0).stop 2024-04-02T04:54:43.051 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.045+0000 7f94c9fa1700 1 -- 172.21.15.152:0/2360444340 shutdown_connections 2024-04-02T04:54:43.051 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.045+0000 7f94c9fa1700 1 --2- 172.21.15.152:0/2360444340 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f94c40fffb0 0x7f94c4102390 unknown :-1 s=CLOSED pgs=183 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:43.051 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.045+0000 7f94c9fa1700 1 --2- 172.21.15.152:0/2360444340 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f94c4066f10 0x7f94c40ffa70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:43.051 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.045+0000 7f94c9fa1700 1 -- 172.21.15.152:0/2360444340 >> 172.21.15.152:0/2360444340 conn(0x7f94c40f7480 msgr2=0x7f94c40f98b0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:54:43.051 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.046+0000 7f94c9fa1700 1 -- 172.21.15.152:0/2360444340 shutdown_connections 2024-04-02T04:54:43.052 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.046+0000 7f94c9fa1700 1 -- 172.21.15.152:0/2360444340 wait complete. 2024-04-02T04:54:43.052 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.046+0000 7f94c9fa1700 1 Processor -- start 2024-04-02T04:54:43.052 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.046+0000 7f94c9fa1700 1 -- start start 2024-04-02T04:54:43.052 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.046+0000 7f94c9fa1700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f94c4066f10 0x7f94c410d960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:43.052 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.046+0000 7f94c9fa1700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f94c40fffb0 0x7f94c410dea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:43.052 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.046+0000 7f94c9fa1700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f94c410e530 con 0x7f94c40fffb0 2024-04-02T04:54:43.052 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.046+0000 7f94c9fa1700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f94c4112ed0 con 0x7f94c4066f10 2024-04-02T04:54:43.052 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.047+0000 7f94bbfff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f94c40fffb0 0x7f94c410dea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:43.052 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.047+0000 7f94bbfff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f94c40fffb0 0x7f94c410dea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:49398/0 (socket says 172.21.15.152:49398) 2024-04-02T04:54:43.052 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.047+0000 7f94bbfff700 1 -- 172.21.15.152:0/4207902744 learned_addr learned my addr 172.21.15.152:0/4207902744 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:54:43.052 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.047+0000 7f94c37fe700 1 --2- 172.21.15.152:0/4207902744 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f94c4066f10 0x7f94c410d960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:43.052 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.047+0000 7f94bbfff700 1 -- 172.21.15.152:0/4207902744 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f94c4066f10 msgr2=0x7f94c410d960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:43.052 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.047+0000 7f94bbfff700 1 --2- 172.21.15.152:0/4207902744 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f94c4066f10 0x7f94c410d960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:43.052 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.047+0000 7f94bbfff700 1 -- 172.21.15.152:0/4207902744 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f94ac005040 con 0x7f94c40fffb0 2024-04-02T04:54:43.052 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.047+0000 7f94bbfff700 1 --2- 172.21.15.152:0/4207902744 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f94c40fffb0 0x7f94c410dea0 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f94ac00b3b0 tx=0x7f94ac009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:54:43.053 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.047+0000 7f94c0ff9700 1 -- 172.21.15.152:0/4207902744 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f94ac00e040 con 0x7f94c40fffb0 2024-04-02T04:54:43.053 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.047+0000 7f94c0ff9700 1 -- 172.21.15.152:0/4207902744 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f94ac00bbf0 con 0x7f94c40fffb0 2024-04-02T04:54:43.053 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.047+0000 7f94c9fa1700 1 -- 172.21.15.152:0/4207902744 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f94c41130d0 con 0x7f94c40fffb0 2024-04-02T04:54:43.053 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.047+0000 7f94c0ff9700 1 -- 172.21.15.152:0/4207902744 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f94ac01c630 con 0x7f94c40fffb0 2024-04-02T04:54:43.053 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.047+0000 7f94c9fa1700 1 -- 172.21.15.152:0/4207902744 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f94c4113640 con 0x7f94c40fffb0 2024-04-02T04:54:43.054 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.048+0000 7f94c9fa1700 1 -- 172.21.15.152:0/4207902744 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f94c404c710 con 0x7f94c40fffb0 2024-04-02T04:54:43.054 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.049+0000 7f94c0ff9700 1 -- 172.21.15.152:0/4207902744 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f94ac00bd60 con 0x7f94c40fffb0 2024-04-02T04:54:43.054 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.049+0000 7f94c0ff9700 1 --2- 172.21.15.152:0/4207902744 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f94b00692a0 0x7f94b006b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:43.054 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.049+0000 7f94c0ff9700 1 -- 172.21.15.152:0/4207902744 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7f94ac087a50 con 0x7f94c40fffb0 2024-04-02T04:54:43.054 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.049+0000 7f94c37fe700 1 --2- 172.21.15.152:0/4207902744 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f94b00692a0 0x7f94b006b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:43.054 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.049+0000 7f94c37fe700 1 --2- 172.21.15.152:0/4207902744 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f94b00692a0 0x7f94b006b750 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7f94c4066c50 tx=0x7f94b40091a0 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:54:43.057 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.053+0000 7f94c0ff9700 1 -- 172.21.15.152:0/4207902744 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f94ac057bf0 con 0x7f94c40fffb0 2024-04-02T04:54:43.317 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.313+0000 7f94c9fa1700 1 -- 172.21.15.152:0/4207902744 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]} v 0) v1 -- 0x7f94c41059a0 con 0x7f94c40fffb0 2024-04-02T04:54:43.320 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.317+0000 7f94c0ff9700 1 -- 172.21.15.152:0/4207902744 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]=0 v14) v1 ==== 170+0+59 (secure 0 0 0) 0x7f94ac057780 con 0x7f94c40fffb0 2024-04-02T04:54:43.321 INFO:teuthology.orchestra.run.smithi152.stdout:[client.0] 2024-04-02T04:54:43.321 INFO:teuthology.orchestra.run.smithi152.stdout: key = AQCTjwtmFSvHEhAAZ3u+RJknPbUbGnMSC4FNyw== 2024-04-02T04:54:43.323 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.319+0000 7f94c9fa1700 1 -- 172.21.15.152:0/4207902744 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f94b00692a0 msgr2=0x7f94b006b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:43.323 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.319+0000 7f94c9fa1700 1 --2- 172.21.15.152:0/4207902744 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f94b00692a0 0x7f94b006b750 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 crypto rx=0x7f94c4066c50 tx=0x7f94b40091a0 comp rx=0 tx=0).stop 2024-04-02T04:54:43.323 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.319+0000 7f94c9fa1700 1 -- 172.21.15.152:0/4207902744 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f94c40fffb0 msgr2=0x7f94c410dea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:43.323 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.319+0000 7f94c9fa1700 1 --2- 172.21.15.152:0/4207902744 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f94c40fffb0 0x7f94c410dea0 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 crypto rx=0x7f94ac00b3b0 tx=0x7f94ac009e30 comp rx=0 tx=0).stop 2024-04-02T04:54:43.323 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.319+0000 7f94c9fa1700 1 -- 172.21.15.152:0/4207902744 shutdown_connections 2024-04-02T04:54:43.323 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.319+0000 7f94c9fa1700 1 --2- 172.21.15.152:0/4207902744 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f94b00692a0 0x7f94b006b750 unknown :-1 s=CLOSED pgs=65 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:43.323 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.319+0000 7f94c9fa1700 1 --2- 172.21.15.152:0/4207902744 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f94c4066f10 0x7f94c410d960 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:43.323 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.319+0000 7f94c9fa1700 1 --2- 172.21.15.152:0/4207902744 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f94c40fffb0 0x7f94c410dea0 unknown :-1 s=CLOSED pgs=184 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:43.324 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.319+0000 7f94c9fa1700 1 -- 172.21.15.152:0/4207902744 >> 172.21.15.152:0/4207902744 conn(0x7f94c40f7480 msgr2=0x7f94c40f78a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:54:43.324 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.319+0000 7f94c9fa1700 1 -- 172.21.15.152:0/4207902744 shutdown_connections 2024-04-02T04:54:43.324 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:43.319+0000 7f94c9fa1700 1 -- 172.21.15.152:0/4207902744 wait complete. 2024-04-02T04:54:43.661 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:43 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/4207902744' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T04:54:43.661 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:43 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/4207902744' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-02T04:54:43.919 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-02T04:54:43.919 DEBUG:teuthology.orchestra.run.smithi152:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-02T04:54:43.919 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-02T04:54:43.961 DEBUG:teuthology.orchestra.run.smithi154:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-02T04:54:43.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:43 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/4207902744' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T04:54:43.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:43 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/4207902744' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-02T04:54:44.227 INFO:teuthology.orchestra.run.smithi154.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi154/config 2024-04-02T04:54:44.676 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:44 smithi154 ceph-mon[35729]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:54:45.006 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:44 smithi152 ceph-mon[29073]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:54:45.780 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.774+0000 7f89bb8c3700 1 -- 172.21.15.154:0/1893389376 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f89b40fbcd0 msgr2=0x7f89b40fc0e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:45.780 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.774+0000 7f89bb8c3700 1 --2- 172.21.15.154:0/1893389376 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f89b40fbcd0 0x7f89b40fc0e0 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 crypto rx=0x7f89b0004940 tx=0x7f89b0009e30 comp rx=0 tx=0).stop 2024-04-02T04:54:45.780 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.774+0000 7f89bb8c3700 1 -- 172.21.15.154:0/1893389376 shutdown_connections 2024-04-02T04:54:45.780 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.774+0000 7f89bb8c3700 1 --2- 172.21.15.154:0/1893389376 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f89b40fcf80 0x7f89b40fd3f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:45.780 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.774+0000 7f89bb8c3700 1 --2- 172.21.15.154:0/1893389376 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f89b40fbcd0 0x7f89b40fc0e0 unknown :-1 s=CLOSED pgs=32 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:45.780 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.774+0000 7f89bb8c3700 1 -- 172.21.15.154:0/1893389376 >> 172.21.15.154:0/1893389376 conn(0x7f89b40f7450 msgr2=0x7f89b40f9860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:54:45.780 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.775+0000 7f89bb8c3700 1 -- 172.21.15.154:0/1893389376 shutdown_connections 2024-04-02T04:54:45.780 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.775+0000 7f89bb8c3700 1 -- 172.21.15.154:0/1893389376 wait complete. 2024-04-02T04:54:45.780 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.775+0000 7f89bb8c3700 1 Processor -- start 2024-04-02T04:54:45.780 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.775+0000 7f89bb8c3700 1 -- start start 2024-04-02T04:54:45.781 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.775+0000 7f89bb8c3700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f89b40fbcd0 0x7f89b410c810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:45.781 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.775+0000 7f89bb8c3700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f89b40fcf80 0x7f89b410cd50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:45.781 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.775+0000 7f89bb8c3700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f89b410d3e0 con 0x7f89b40fbcd0 2024-04-02T04:54:45.781 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.775+0000 7f89bb8c3700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f89b4111d80 con 0x7f89b40fcf80 2024-04-02T04:54:45.781 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.775+0000 7f89b8e5e700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f89b40fcf80 0x7f89b410cd50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:45.781 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.775+0000 7f89b8e5e700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f89b40fcf80 0x7f89b410cd50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.154:39658/0 (socket says 172.21.15.154:39658) 2024-04-02T04:54:45.781 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.775+0000 7f89b8e5e700 1 -- 172.21.15.154:0/4256145668 learned_addr learned my addr 172.21.15.154:0/4256145668 (peer_addr_for_me v2:172.21.15.154:0/0) 2024-04-02T04:54:45.781 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.775+0000 7f89b965f700 1 --2- 172.21.15.154:0/4256145668 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f89b40fbcd0 0x7f89b410c810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:45.781 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.776+0000 7f89b8e5e700 1 -- 172.21.15.154:0/4256145668 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f89b40fbcd0 msgr2=0x7f89b410c810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:45.781 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.776+0000 7f89b8e5e700 1 --2- 172.21.15.154:0/4256145668 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f89b40fbcd0 0x7f89b410c810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:45.781 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.776+0000 7f89b8e5e700 1 -- 172.21.15.154:0/4256145668 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f89b0005040 con 0x7f89b40fcf80 2024-04-02T04:54:45.781 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.776+0000 7f89b8e5e700 1 --2- 172.21.15.154:0/4256145668 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f89b40fcf80 0x7f89b410cd50 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7f89a400b790 tx=0x7f89a4009250 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:54:45.781 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.776+0000 7f89a9ffb700 1 -- 172.21.15.154:0/4256145668 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f89a400f6f0 con 0x7f89b40fcf80 2024-04-02T04:54:45.781 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.776+0000 7f89a9ffb700 1 -- 172.21.15.154:0/4256145668 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f89a400fd20 con 0x7f89b40fcf80 2024-04-02T04:54:45.781 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.776+0000 7f89bb8c3700 1 -- 172.21.15.154:0/4256145668 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f89b4111fe0 con 0x7f89b40fcf80 2024-04-02T04:54:45.781 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.776+0000 7f89a9ffb700 1 -- 172.21.15.154:0/4256145668 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f89a40103a0 con 0x7f89b40fcf80 2024-04-02T04:54:45.782 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.776+0000 7f89bb8c3700 1 -- 172.21.15.154:0/4256145668 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f89b4112520 con 0x7f89b40fcf80 2024-04-02T04:54:45.782 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.777+0000 7f89bb8c3700 1 -- 172.21.15.154:0/4256145668 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f89b404c710 con 0x7f89b40fcf80 2024-04-02T04:54:45.783 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.777+0000 7f89a9ffb700 1 -- 172.21.15.154:0/4256145668 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f89a4011400 con 0x7f89b40fcf80 2024-04-02T04:54:45.783 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.778+0000 7f89a9ffb700 1 --2- 172.21.15.154:0/4256145668 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f89a00692a0 0x7f89a006b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:45.783 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.778+0000 7f89a9ffb700 1 -- 172.21.15.154:0/4256145668 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7f89a4086c50 con 0x7f89b40fcf80 2024-04-02T04:54:45.783 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.778+0000 7f89b965f700 1 --2- 172.21.15.154:0/4256145668 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f89a00692a0 0x7f89a006b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:45.783 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.778+0000 7f89b965f700 1 --2- 172.21.15.154:0/4256145668 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f89a00692a0 0x7f89a006b750 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7f89b0004890 tx=0x7f89b0009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:54:45.785 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:45.782+0000 7f89a9ffb700 1 -- 172.21.15.154:0/4256145668 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f89a4056d50 con 0x7f89b40fcf80 2024-04-02T04:54:46.047 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:46.043+0000 7f89bb8c3700 1 -- 172.21.15.154:0/4256145668 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]} v 0) v1 -- 0x7f89b4104840 con 0x7f89b40fcf80 2024-04-02T04:54:46.051 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:46.047+0000 7f89a9ffb700 1 -- 172.21.15.154:0/4256145668 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]=0 v15) v1 ==== 170+0+59 (secure 0 0 0) 0x7f89a40568e0 con 0x7f89b40fcf80 2024-04-02T04:54:46.097 INFO:teuthology.orchestra.run.smithi154.stdout:[client.1] 2024-04-02T04:54:46.097 INFO:teuthology.orchestra.run.smithi154.stdout: key = AQCWjwtmZdWwAhAAeqgGNE9reGubiEfo9H1+EA== 2024-04-02T04:54:46.098 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:46.049+0000 7f89bb8c3700 1 -- 172.21.15.154:0/4256145668 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f89a00692a0 msgr2=0x7f89a006b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:46.098 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:46.049+0000 7f89bb8c3700 1 --2- 172.21.15.154:0/4256145668 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f89a00692a0 0x7f89a006b750 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 crypto rx=0x7f89b0004890 tx=0x7f89b0009e30 comp rx=0 tx=0).stop 2024-04-02T04:54:46.098 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:46.049+0000 7f89bb8c3700 1 -- 172.21.15.154:0/4256145668 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f89b40fcf80 msgr2=0x7f89b410cd50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:46.098 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:46.049+0000 7f89bb8c3700 1 --2- 172.21.15.154:0/4256145668 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f89b40fcf80 0x7f89b410cd50 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 crypto rx=0x7f89a400b790 tx=0x7f89a4009250 comp rx=0 tx=0).stop 2024-04-02T04:54:46.098 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:46.049+0000 7f89bb8c3700 1 -- 172.21.15.154:0/4256145668 shutdown_connections 2024-04-02T04:54:46.098 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:46.049+0000 7f89bb8c3700 1 --2- 172.21.15.154:0/4256145668 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f89a00692a0 0x7f89a006b750 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:46.098 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:46.049+0000 7f89bb8c3700 1 --2- 172.21.15.154:0/4256145668 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f89b40fbcd0 0x7f89b410c810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:46.098 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:46.049+0000 7f89bb8c3700 1 --2- 172.21.15.154:0/4256145668 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f89b40fcf80 0x7f89b410cd50 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:46.098 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:46.049+0000 7f89bb8c3700 1 -- 172.21.15.154:0/4256145668 >> 172.21.15.154:0/4256145668 conn(0x7f89b40f7450 msgr2=0x7f89b40f8a20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:54:46.099 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:46.049+0000 7f89bb8c3700 1 -- 172.21.15.154:0/4256145668 shutdown_connections 2024-04-02T04:54:46.099 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:54:46.050+0000 7f89bb8c3700 1 -- 172.21.15.154:0/4256145668 wait complete. 2024-04-02T04:54:46.637 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-04-02T04:54:46.637 DEBUG:teuthology.orchestra.run.smithi154:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-04-02T04:54:46.637 DEBUG:teuthology.orchestra.run.smithi154:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-04-02T04:54:46.720 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-02T04:54:46.720 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-02T04:54:46.721 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph mgr dump --format=json 2024-04-02T04:54:46.985 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:54:47.340 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:47 smithi152 ceph-mon[29073]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:54:47.340 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:47 smithi152 ceph-mon[29073]: from='client.? 172.21.15.154:0/4256145668' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T04:54:47.340 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:47 smithi152 ceph-mon[29073]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T04:54:47.340 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:47 smithi152 ceph-mon[29073]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-02T04:54:47.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:47 smithi154 ceph-mon[35729]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:54:47.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:47 smithi154 ceph-mon[35729]: from='client.? 172.21.15.154:0/4256145668' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T04:54:47.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:47 smithi154 ceph-mon[35729]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-02T04:54:47.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:47 smithi154 ceph-mon[35729]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-02T04:54:48.597 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.591+0000 7fed5803e700 1 -- 172.21.15.152:0/1228646248 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fed501015e0 msgr2=0x7fed500fb3b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:48.597 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.591+0000 7fed5803e700 1 --2- 172.21.15.152:0/1228646248 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fed501015e0 0x7fed500fb3b0 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 crypto rx=0x7fed40004940 tx=0x7fed40009e30 comp rx=0 tx=0).stop 2024-04-02T04:54:48.597 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.591+0000 7fed5803e700 1 -- 172.21.15.152:0/1228646248 shutdown_connections 2024-04-02T04:54:48.597 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.591+0000 7fed5803e700 1 --2- 172.21.15.152:0/1228646248 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fed501015e0 0x7fed500fb3b0 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:48.597 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.591+0000 7fed5803e700 1 --2- 172.21.15.152:0/1228646248 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fed50100c40 0x7fed50101010 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:48.597 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.591+0000 7fed5803e700 1 -- 172.21.15.152:0/1228646248 >> 172.21.15.152:0/1228646248 conn(0x7fed50072310 msgr2=0x7fed50072710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:54:48.597 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.592+0000 7fed5803e700 1 -- 172.21.15.152:0/1228646248 shutdown_connections 2024-04-02T04:54:48.597 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.592+0000 7fed5803e700 1 -- 172.21.15.152:0/1228646248 wait complete. 2024-04-02T04:54:48.597 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.592+0000 7fed5803e700 1 Processor -- start 2024-04-02T04:54:48.598 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.592+0000 7fed5803e700 1 -- start start 2024-04-02T04:54:48.598 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.592+0000 7fed5803e700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fed50100c40 0x7fed5010ed80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:48.598 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.592+0000 7fed5803e700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fed501015e0 0x7fed50109f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:48.598 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.592+0000 7fed5803e700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fed5010a520 con 0x7fed50100c40 2024-04-02T04:54:48.598 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.592+0000 7fed5803e700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fed5010a690 con 0x7fed501015e0 2024-04-02T04:54:48.598 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.592+0000 7fed55dda700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fed50100c40 0x7fed5010ed80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:48.598 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.592+0000 7fed55dda700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fed50100c40 0x7fed5010ed80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:49416/0 (socket says 172.21.15.152:49416) 2024-04-02T04:54:48.598 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.592+0000 7fed55dda700 1 -- 172.21.15.152:0/2818096371 learned_addr learned my addr 172.21.15.152:0/2818096371 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:54:48.598 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.593+0000 7fed555d9700 1 --2- 172.21.15.152:0/2818096371 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fed501015e0 0x7fed50109f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:48.598 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.593+0000 7fed55dda700 1 -- 172.21.15.152:0/2818096371 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fed501015e0 msgr2=0x7fed50109f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:48.598 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.593+0000 7fed55dda700 1 --2- 172.21.15.152:0/2818096371 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fed501015e0 0x7fed50109f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:48.598 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.593+0000 7fed55dda700 1 -- 172.21.15.152:0/2818096371 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fed40005040 con 0x7fed50100c40 2024-04-02T04:54:48.599 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.593+0000 7fed55dda700 1 --2- 172.21.15.152:0/2818096371 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fed50100c40 0x7fed5010ed80 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7fed4c00b790 tx=0x7fed4c009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:54:48.599 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.593+0000 7fed467fc700 1 -- 172.21.15.152:0/2818096371 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fed4c00f770 con 0x7fed50100c40 2024-04-02T04:54:48.599 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.593+0000 7fed467fc700 1 -- 172.21.15.152:0/2818096371 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fed4c00fda0 con 0x7fed50100c40 2024-04-02T04:54:48.599 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.593+0000 7fed5803e700 1 -- 172.21.15.152:0/2818096371 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fed5010a970 con 0x7fed50100c40 2024-04-02T04:54:48.599 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.593+0000 7fed467fc700 1 -- 172.21.15.152:0/2818096371 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fed4c00d310 con 0x7fed50100c40 2024-04-02T04:54:48.599 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.593+0000 7fed5803e700 1 -- 172.21.15.152:0/2818096371 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fed50126dd0 con 0x7fed50100c40 2024-04-02T04:54:48.600 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.594+0000 7fed5803e700 1 -- 172.21.15.152:0/2818096371 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fed50104240 con 0x7fed50100c40 2024-04-02T04:54:48.600 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.595+0000 7fed467fc700 1 -- 172.21.15.152:0/2818096371 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7fed4c010420 con 0x7fed50100c40 2024-04-02T04:54:48.600 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.595+0000 7fed467fc700 1 --2- 172.21.15.152:0/2818096371 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fed3c0692a0 0x7fed3c06b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:48.600 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.595+0000 7fed467fc700 1 -- 172.21.15.152:0/2818096371 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7fed4c086d50 con 0x7fed50100c40 2024-04-02T04:54:48.600 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.595+0000 7fed555d9700 1 --2- 172.21.15.152:0/2818096371 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fed3c0692a0 0x7fed3c06b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:48.600 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.595+0000 7fed555d9700 1 --2- 172.21.15.152:0/2818096371 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fed3c0692a0 0x7fed3c06b750 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7fed5010b630 tx=0x7fed40009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:54:48.603 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.599+0000 7fed467fc700 1 -- 172.21.15.152:0/2818096371 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fed4c053460 con 0x7fed50100c40 2024-04-02T04:54:48.842 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.839+0000 7fed5803e700 1 -- 172.21.15.152:0/2818096371 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "mgr dump", "format": "json"} v 0) v1 -- 0x7fed500fc3c0 con 0x7fed50100c40 2024-04-02T04:54:48.846 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.842+0000 7fed467fc700 1 -- 172.21.15.152:0/2818096371 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "mgr dump", "format": "json"}]=0 v19) v1 ==== 74+0+167176 (secure 0 0 0) 0x7fed4c056a80 con 0x7fed50100c40 2024-04-02T04:54:48.848 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:54:48.850 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.844+0000 7fed5803e700 1 -- 172.21.15.152:0/2818096371 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fed3c0692a0 msgr2=0x7fed3c06b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:48.850 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.844+0000 7fed5803e700 1 --2- 172.21.15.152:0/2818096371 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fed3c0692a0 0x7fed3c06b750 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 crypto rx=0x7fed5010b630 tx=0x7fed40009e30 comp rx=0 tx=0).stop 2024-04-02T04:54:48.850 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.844+0000 7fed5803e700 1 -- 172.21.15.152:0/2818096371 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fed50100c40 msgr2=0x7fed5010ed80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:48.850 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.844+0000 7fed5803e700 1 --2- 172.21.15.152:0/2818096371 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fed50100c40 0x7fed5010ed80 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 crypto rx=0x7fed4c00b790 tx=0x7fed4c009250 comp rx=0 tx=0).stop 2024-04-02T04:54:48.850 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.844+0000 7fed5803e700 1 -- 172.21.15.152:0/2818096371 shutdown_connections 2024-04-02T04:54:48.850 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.844+0000 7fed5803e700 1 --2- 172.21.15.152:0/2818096371 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fed3c0692a0 0x7fed3c06b750 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:48.850 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.844+0000 7fed5803e700 1 --2- 172.21.15.152:0/2818096371 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fed50100c40 0x7fed5010ed80 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:48.850 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.844+0000 7fed5803e700 1 --2- 172.21.15.152:0/2818096371 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fed501015e0 0x7fed50109f00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:48.850 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.844+0000 7fed5803e700 1 -- 172.21.15.152:0/2818096371 >> 172.21.15.152:0/2818096371 conn(0x7fed50072310 msgr2=0x7fed500fde00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:54:48.850 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.844+0000 7fed5803e700 1 -- 172.21.15.152:0/2818096371 shutdown_connections 2024-04-02T04:54:48.850 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:48.845+0000 7fed5803e700 1 -- 172.21.15.152:0/2818096371 wait complete. 2024-04-02T04:54:49.140 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:49 smithi152 ceph-mon[29073]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:54:49.141 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:49 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/2818096371' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T04:54:49.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:49 smithi154 ceph-mon[35729]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:54:49.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:49 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/2818096371' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-02T04:54:49.502 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":19,"active_gid":14210,"active_name":"smithi152.jivdcl","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6800","nonce":3117501261},{"type":"v1","addr":"172.21.15.152:6801","nonce":3117501261}]},"active_addr":"172.21.15.152:6801/3117501261","active_change":"2024-04-02T04:51:09.950019+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":14228,"name":"smithi154.pplnxz","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.152:8443/","prometheus":"http://172.21.15.152:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":5,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.152:0","nonce":32509388}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.152:0","nonce":3989692598}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.152:0","nonce":1445035452}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.152:0","nonce":2422977248}]}]} 2024-04-02T04:54:49.505 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-02T04:54:49.506 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-02T04:54:49.506 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-02T04:54:49.765 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:54:51.334 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:51 smithi154 ceph-mon[35729]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:54:51.371 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:51 smithi152 ceph-mon[29073]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:54:51.584 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.580+0000 7f2a49430700 1 -- 172.21.15.152:0/1384023815 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2a44102e10 msgr2=0x7f2a441031e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:51.584 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.580+0000 7f2a49430700 1 --2- 172.21.15.152:0/1384023815 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2a44102e10 0x7f2a441031e0 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 crypto rx=0x7f2a34004940 tx=0x7f2a34009e30 comp rx=0 tx=0).stop 2024-04-02T04:54:51.584 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.580+0000 7f2a49430700 1 -- 172.21.15.152:0/1384023815 shutdown_connections 2024-04-02T04:54:51.585 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.580+0000 7f2a49430700 1 --2- 172.21.15.152:0/1384023815 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2a441037b0 0x7f2a440fce20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:51.585 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.580+0000 7f2a49430700 1 --2- 172.21.15.152:0/1384023815 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2a44102e10 0x7f2a441031e0 unknown :-1 s=CLOSED pgs=187 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:51.585 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.580+0000 7f2a49430700 1 -- 172.21.15.152:0/1384023815 >> 172.21.15.152:0/1384023815 conn(0x7f2a440f8980 msgr2=0x7f2a440fad90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:54:51.585 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.580+0000 7f2a49430700 1 -- 172.21.15.152:0/1384023815 shutdown_connections 2024-04-02T04:54:51.585 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.580+0000 7f2a49430700 1 -- 172.21.15.152:0/1384023815 wait complete. 2024-04-02T04:54:51.585 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.580+0000 7f2a49430700 1 Processor -- start 2024-04-02T04:54:51.585 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.581+0000 7f2a49430700 1 -- start start 2024-04-02T04:54:51.587 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.581+0000 7f2a49430700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2a44102e10 0x7f2a44113ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:51.588 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.581+0000 7f2a49430700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2a441037b0 0x7f2a441140e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:51.588 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.581+0000 7f2a49430700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2a44114730 con 0x7f2a441037b0 2024-04-02T04:54:51.588 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.581+0000 7f2a49430700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2a4410db40 con 0x7f2a44102e10 2024-04-02T04:54:51.588 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.581+0000 7f2a3a7fc700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2a441037b0 0x7f2a441140e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:51.588 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.581+0000 7f2a3a7fc700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2a441037b0 0x7f2a441140e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:36538/0 (socket says 172.21.15.152:36538) 2024-04-02T04:54:51.588 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.581+0000 7f2a3a7fc700 1 -- 172.21.15.152:0/316675285 learned_addr learned my addr 172.21.15.152:0/316675285 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:54:51.588 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.581+0000 7f2a42ffd700 1 --2- 172.21.15.152:0/316675285 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2a44102e10 0x7f2a44113ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:51.588 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.581+0000 7f2a3a7fc700 1 -- 172.21.15.152:0/316675285 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2a44102e10 msgr2=0x7f2a44113ba0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:51.588 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.581+0000 7f2a3a7fc700 1 --2- 172.21.15.152:0/316675285 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2a44102e10 0x7f2a44113ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:51.588 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.581+0000 7f2a3a7fc700 1 -- 172.21.15.152:0/316675285 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2a34005040 con 0x7f2a441037b0 2024-04-02T04:54:51.588 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.582+0000 7f2a3a7fc700 1 --2- 172.21.15.152:0/316675285 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2a441037b0 0x7f2a441140e0 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7f2a2c00c800 tx=0x7f2a2c004470 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:54:51.588 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.582+0000 7f2a3bfff700 1 -- 172.21.15.152:0/316675285 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f2a2c00a760 con 0x7f2a441037b0 2024-04-02T04:54:51.588 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.582+0000 7f2a3bfff700 1 -- 172.21.15.152:0/316675285 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f2a2c00ad90 con 0x7f2a441037b0 2024-04-02T04:54:51.589 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.582+0000 7f2a49430700 1 -- 172.21.15.152:0/316675285 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2a4410de20 con 0x7f2a441037b0 2024-04-02T04:54:51.589 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.582+0000 7f2a3bfff700 1 -- 172.21.15.152:0/316675285 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f2a2c0113a0 con 0x7f2a441037b0 2024-04-02T04:54:51.589 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.582+0000 7f2a49430700 1 -- 172.21.15.152:0/316675285 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2a4410e3e0 con 0x7f2a441037b0 2024-04-02T04:54:51.589 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.583+0000 7f2a49430700 1 -- 172.21.15.152:0/316675285 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2a44100120 con 0x7f2a441037b0 2024-04-02T04:54:51.589 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.583+0000 7f2a3bfff700 1 -- 172.21.15.152:0/316675285 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f2a2c012400 con 0x7f2a441037b0 2024-04-02T04:54:51.589 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.584+0000 7f2a3bfff700 1 --2- 172.21.15.152:0/316675285 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2a300692a0 0x7f2a3006b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:51.589 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.584+0000 7f2a3bfff700 1 -- 172.21.15.152:0/316675285 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7f2a2c086e90 con 0x7f2a441037b0 2024-04-02T04:54:51.589 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.584+0000 7f2a42ffd700 1 --2- 172.21.15.152:0/316675285 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2a300692a0 0x7f2a3006b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:51.589 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.584+0000 7f2a42ffd700 1 --2- 172.21.15.152:0/316675285 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2a300692a0 0x7f2a3006b750 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f2a34004f90 tx=0x7f2a34009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:54:51.592 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.588+0000 7f2a3bfff700 1 -- 172.21.15.152:0/316675285 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f2a2c053500 con 0x7f2a441037b0 2024-04-02T04:54:51.781 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.777+0000 7f2a49430700 1 -- 172.21.15.152:0/316675285 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f2a4410f1f0 con 0x7f2a441037b0 2024-04-02T04:54:51.781 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.778+0000 7f2a3bfff700 1 -- 172.21.15.152:0/316675285 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v40) v1 ==== 74+0+11112 (secure 0 0 0) 0x7f2a2c056b20 con 0x7f2a441037b0 2024-04-02T04:54:51.782 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:54:51.782 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":40,"fsid":"2e0055d0-f0ac-11ee-b647-cb9ed24678a4","created":"2024-04-02T04:48:59.746166+0000","modified":"2024-04-02T04:54:33.124544+0000","last_up_change":"2024-04-02T04:54:32.119050+0000","last_in_change":"2024-04-02T04:54:13.064706+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":14,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":6,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-02T04:53:20.274250+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"3f25c359-f643-4dfb-ba7b-30bb1a2dfcb2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":718734380},{"type":"v1","addr":"172.21.15.152:6803","nonce":718734380}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":718734380},{"type":"v1","addr":"172.21.15.152:6805","nonce":718734380}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":718734380},{"type":"v1","addr":"172.21.15.152:6809","nonce":718734380}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":718734380},{"type":"v1","addr":"172.21.15.152:6807","nonce":718734380}]},"public_addr":"172.21.15.152:6803/718734380","cluster_addr":"172.21.15.152:6805/718734380","heartbeat_back_addr":"172.21.15.152:6809/718734380","heartbeat_front_addr":"172.21.15.152:6807/718734380","state":["exists","up"]},{"osd":1,"uuid":"d4555f9e-fc0e-4266-ae65-3c197d3fa02c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":29,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":782616861},{"type":"v1","addr":"172.21.15.152:6811","nonce":782616861}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":782616861},{"type":"v1","addr":"172.21.15.152:6813","nonce":782616861}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":782616861},{"type":"v1","addr":"172.21.15.152:6817","nonce":782616861}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":782616861},{"type":"v1","addr":"172.21.15.152:6815","nonce":782616861}]},"public_addr":"172.21.15.152:6811/782616861","cluster_addr":"172.21.15.152:6813/782616861","heartbeat_back_addr":"172.21.15.152:6817/782616861","heartbeat_front_addr":"172.21.15.152:6815/782616861","state":["exists","up"]},{"osd":2,"uuid":"caa0a912-a9d5-45e7-afe2-51beaf0f1742","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":1566571469},{"type":"v1","addr":"172.21.15.152:6819","nonce":1566571469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":1566571469},{"type":"v1","addr":"172.21.15.152:6821","nonce":1566571469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":1566571469},{"type":"v1","addr":"172.21.15.152:6825","nonce":1566571469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":1566571469},{"type":"v1","addr":"172.21.15.152:6823","nonce":1566571469}]},"public_addr":"172.21.15.152:6819/1566571469","cluster_addr":"172.21.15.152:6821/1566571469","heartbeat_back_addr":"172.21.15.152:6825/1566571469","heartbeat_front_addr":"172.21.15.152:6823/1566571469","state":["exists","up"]},{"osd":3,"uuid":"84a2d337-6ac8-41e4-a922-a6e94c1e9067","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6800","nonce":3366219775},{"type":"v1","addr":"172.21.15.154:6801","nonce":3366219775}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6802","nonce":3366219775},{"type":"v1","addr":"172.21.15.154:6803","nonce":3366219775}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6806","nonce":3366219775},{"type":"v1","addr":"172.21.15.154:6807","nonce":3366219775}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6804","nonce":3366219775},{"type":"v1","addr":"172.21.15.154:6805","nonce":3366219775}]},"public_addr":"172.21.15.154:6801/3366219775","cluster_addr":"172.21.15.154:6803/3366219775","heartbeat_back_addr":"172.21.15.154:6807/3366219775","heartbeat_front_addr":"172.21.15.154:6805/3366219775","state":["exists","up"]},{"osd":4,"uuid":"d547308c-898c-4165-9762-c8fd4170dc39","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":34,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6808","nonce":220818867},{"type":"v1","addr":"172.21.15.154:6809","nonce":220818867}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6810","nonce":220818867},{"type":"v1","addr":"172.21.15.154:6811","nonce":220818867}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6814","nonce":220818867},{"type":"v1","addr":"172.21.15.154:6815","nonce":220818867}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6812","nonce":220818867},{"type":"v1","addr":"172.21.15.154:6813","nonce":220818867}]},"public_addr":"172.21.15.154:6809/220818867","cluster_addr":"172.21.15.154:6811/220818867","heartbeat_back_addr":"172.21.15.154:6815/220818867","heartbeat_front_addr":"172.21.15.154:6813/220818867","state":["exists","up"]},{"osd":5,"uuid":"c3d3760a-08a7-44f2-8288-ccc5d08df057","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6816","nonce":1178927498},{"type":"v1","addr":"172.21.15.154:6817","nonce":1178927498}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6818","nonce":1178927498},{"type":"v1","addr":"172.21.15.154:6819","nonce":1178927498}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6822","nonce":1178927498},{"type":"v1","addr":"172.21.15.154:6823","nonce":1178927498}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6820","nonce":1178927498},{"type":"v1","addr":"172.21.15.154:6821","nonce":1178927498}]},"public_addr":"172.21.15.154:6817/1178927498","cluster_addr":"172.21.15.154:6819/1178927498","heartbeat_back_addr":"172.21.15.154:6823/1178927498","heartbeat_front_addr":"172.21.15.154:6821/1178927498","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:52:20.472492+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:52:49.979314+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:53:17.640952+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:53:40.392841+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:54:05.761710+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:54:30.235521+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.152:0/3027168373":"2024-04-03T04:51:09.949838+0000","172.21.15.152:0/2268166004":"2024-04-03T04:51:09.949838+0000","172.21.15.152:6800/2497421829":"2024-04-03T04:51:09.949838+0000","172.21.15.152:6801/2497421829":"2024-04-03T04:51:09.949838+0000","172.21.15.152:0/4188679877":"2024-04-03T04:50:01.902008+0000","172.21.15.152:0/1860988983":"2024-04-03T04:50:01.902008+0000","172.21.15.152:0/1943568671":"2024-04-03T04:49:30.819772+0000","172.21.15.152:0/2491428539":"2024-04-03T04:51:09.949838+0000","172.21.15.152:6801/3544077480":"2024-04-03T04:50:01.902008+0000","172.21.15.152:0/3476288520":"2024-04-03T04:49:30.819772+0000","172.21.15.152:6800/2913738627":"2024-04-03T04:49:30.819772+0000","172.21.15.152:6800/3544077480":"2024-04-03T04:50:01.902008+0000","172.21.15.152:0/3023512755":"2024-04-03T04:49:30.819772+0000","172.21.15.152:6801/2913738627":"2024-04-03T04:49:30.819772+0000","172.21.15.152:0/2947158817":"2024-04-03T04:50:01.902008+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-02T04:54:51.783 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.780+0000 7f2a49430700 1 -- 172.21.15.152:0/316675285 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2a300692a0 msgr2=0x7f2a3006b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:51.784 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.780+0000 7f2a49430700 1 --2- 172.21.15.152:0/316675285 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2a300692a0 0x7f2a3006b750 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 crypto rx=0x7f2a34004f90 tx=0x7f2a34009e30 comp rx=0 tx=0).stop 2024-04-02T04:54:51.784 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.780+0000 7f2a49430700 1 -- 172.21.15.152:0/316675285 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2a441037b0 msgr2=0x7f2a441140e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:51.784 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.780+0000 7f2a49430700 1 --2- 172.21.15.152:0/316675285 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2a441037b0 0x7f2a441140e0 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 crypto rx=0x7f2a2c00c800 tx=0x7f2a2c004470 comp rx=0 tx=0).stop 2024-04-02T04:54:51.784 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.780+0000 7f2a49430700 1 -- 172.21.15.152:0/316675285 shutdown_connections 2024-04-02T04:54:51.784 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.780+0000 7f2a49430700 1 --2- 172.21.15.152:0/316675285 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2a300692a0 0x7f2a3006b750 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:51.784 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.780+0000 7f2a49430700 1 --2- 172.21.15.152:0/316675285 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2a44102e10 0x7f2a44113ba0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:51.784 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.780+0000 7f2a49430700 1 --2- 172.21.15.152:0/316675285 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2a441037b0 0x7f2a441140e0 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:51.784 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.780+0000 7f2a49430700 1 -- 172.21.15.152:0/316675285 >> 172.21.15.152:0/316675285 conn(0x7f2a440f8980 msgr2=0x7f2a44107400 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:54:51.784 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.780+0000 7f2a49430700 1 -- 172.21.15.152:0/316675285 shutdown_connections 2024-04-02T04:54:51.784 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:51.780+0000 7f2a49430700 1 -- 172.21.15.152:0/316675285 wait complete. 2024-04-02T04:54:52.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:52 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/316675285' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T04:54:52.481 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:52 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/316675285' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T04:54:52.507 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-02T04:54:52.507 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph osd dump --format=json 2024-04-02T04:54:52.765 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:54:53.201 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:53 smithi152 ceph-mon[29073]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:54:53.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:53 smithi154 ceph-mon[35729]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:54:54.374 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.369+0000 7f3071fb6700 1 -- 172.21.15.152:0/2132007667 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f306c101d30 msgr2=0x7f306c102100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:54.375 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.369+0000 7f3071fb6700 1 --2- 172.21.15.152:0/2132007667 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f306c101d30 0x7f306c102100 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 crypto rx=0x7f3064009ce0 tx=0x7f3064004d10 comp rx=0 tx=0).stop 2024-04-02T04:54:54.375 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.369+0000 7f3071fb6700 1 -- 172.21.15.152:0/2132007667 shutdown_connections 2024-04-02T04:54:54.375 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.369+0000 7f3071fb6700 1 --2- 172.21.15.152:0/2132007667 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f306c1026d0 0x7f306c0fbd40 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:54.375 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.369+0000 7f3071fb6700 1 --2- 172.21.15.152:0/2132007667 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f306c101d30 0x7f306c102100 unknown :-1 s=CLOSED pgs=189 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:54.375 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.369+0000 7f3071fb6700 1 -- 172.21.15.152:0/2132007667 >> 172.21.15.152:0/2132007667 conn(0x7f306c0f78c0 msgr2=0x7f306c0f9cd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:54:54.375 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.369+0000 7f3071fb6700 1 -- 172.21.15.152:0/2132007667 shutdown_connections 2024-04-02T04:54:54.375 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.369+0000 7f3071fb6700 1 -- 172.21.15.152:0/2132007667 wait complete. 2024-04-02T04:54:54.375 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.370+0000 7f3071fb6700 1 Processor -- start 2024-04-02T04:54:54.375 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.370+0000 7f3071fb6700 1 -- start start 2024-04-02T04:54:54.375 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.370+0000 7f3071fb6700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f306c101d30 0x7f306c10ec50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:54.375 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.370+0000 7f3071fb6700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f306c1026d0 0x7f306c10f190 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:54.375 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.370+0000 7f3071fb6700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f306c10f6d0 con 0x7f306c101d30 2024-04-02T04:54:54.376 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.370+0000 7f3071fb6700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f306c10f810 con 0x7f306c1026d0 2024-04-02T04:54:54.376 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.370+0000 7f306b7fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f306c101d30 0x7f306c10ec50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:54.376 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.370+0000 7f306b7fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f306c101d30 0x7f306c10ec50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:36560/0 (socket says 172.21.15.152:36560) 2024-04-02T04:54:54.376 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.370+0000 7f306b7fe700 1 -- 172.21.15.152:0/1743190394 learned_addr learned my addr 172.21.15.152:0/1743190394 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:54:54.376 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.370+0000 7f306affd700 1 --2- 172.21.15.152:0/1743190394 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f306c1026d0 0x7f306c10f190 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:54.376 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.370+0000 7f306b7fe700 1 -- 172.21.15.152:0/1743190394 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f306c1026d0 msgr2=0x7f306c10f190 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:54.376 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.370+0000 7f306b7fe700 1 --2- 172.21.15.152:0/1743190394 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f306c1026d0 0x7f306c10f190 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:54.376 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.370+0000 7f306b7fe700 1 -- 172.21.15.152:0/1743190394 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3064009940 con 0x7f306c101d30 2024-04-02T04:54:54.376 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.370+0000 7f306affd700 1 --2- 172.21.15.152:0/1743190394 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f306c1026d0 0x7f306c10f190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-02T04:54:54.376 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.371+0000 7f306b7fe700 1 --2- 172.21.15.152:0/1743190394 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f306c101d30 0x7f306c10ec50 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f306400be90 tx=0x7f3064004d10 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:54:54.376 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.371+0000 7f3063fff700 1 -- 172.21.15.152:0/1743190394 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f306400e040 con 0x7f306c101d30 2024-04-02T04:54:54.376 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.371+0000 7f3071fb6700 1 -- 172.21.15.152:0/1743190394 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f306c10fa90 con 0x7f306c101d30 2024-04-02T04:54:54.376 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.371+0000 7f3063fff700 1 -- 172.21.15.152:0/1743190394 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f3064008ea0 con 0x7f306c101d30 2024-04-02T04:54:54.377 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.371+0000 7f3063fff700 1 -- 172.21.15.152:0/1743190394 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f306401c940 con 0x7f306c101d30 2024-04-02T04:54:54.377 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.371+0000 7f3071fb6700 1 -- 172.21.15.152:0/1743190394 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f306c1180c0 con 0x7f306c101d30 2024-04-02T04:54:54.377 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.372+0000 7f3071fb6700 1 -- 172.21.15.152:0/1743190394 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f306c0fd480 con 0x7f306c101d30 2024-04-02T04:54:54.377 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.372+0000 7f3063fff700 1 -- 172.21.15.152:0/1743190394 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f3064012d00 con 0x7f306c101d30 2024-04-02T04:54:54.378 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.372+0000 7f3063fff700 1 --2- 172.21.15.152:0/1743190394 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f30580692a0 0x7f305806b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:54:54.378 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.373+0000 7f3063fff700 1 -- 172.21.15.152:0/1743190394 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7f3064088aa0 con 0x7f306c101d30 2024-04-02T04:54:54.378 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.373+0000 7f306affd700 1 --2- 172.21.15.152:0/1743190394 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f30580692a0 0x7f305806b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:54:54.378 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.373+0000 7f306affd700 1 --2- 172.21.15.152:0/1743190394 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f30580692a0 0x7f305806b750 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f306c066b00 tx=0x7f3054008040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:54:54.380 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.377+0000 7f3063fff700 1 -- 172.21.15.152:0/1743190394 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f3064055090 con 0x7f306c101d30 2024-04-02T04:54:54.573 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.569+0000 7f3071fb6700 1 -- 172.21.15.152:0/1743190394 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7f306c0668b0 con 0x7f306c101d30 2024-04-02T04:54:54.573 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.570+0000 7f3063fff700 1 -- 172.21.15.152:0/1743190394 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v40) v1 ==== 74+0+11112 (secure 0 0 0) 0x7f3064004e10 con 0x7f306c101d30 2024-04-02T04:54:54.574 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:54:54.574 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":40,"fsid":"2e0055d0-f0ac-11ee-b647-cb9ed24678a4","created":"2024-04-02T04:48:59.746166+0000","modified":"2024-04-02T04:54:33.124544+0000","last_up_change":"2024-04-02T04:54:32.119050+0000","last_in_change":"2024-04-02T04:54:13.064706+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":14,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":6,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-02T04:53:20.274250+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"3f25c359-f643-4dfb-ba7b-30bb1a2dfcb2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":718734380},{"type":"v1","addr":"172.21.15.152:6803","nonce":718734380}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":718734380},{"type":"v1","addr":"172.21.15.152:6805","nonce":718734380}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":718734380},{"type":"v1","addr":"172.21.15.152:6809","nonce":718734380}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":718734380},{"type":"v1","addr":"172.21.15.152:6807","nonce":718734380}]},"public_addr":"172.21.15.152:6803/718734380","cluster_addr":"172.21.15.152:6805/718734380","heartbeat_back_addr":"172.21.15.152:6809/718734380","heartbeat_front_addr":"172.21.15.152:6807/718734380","state":["exists","up"]},{"osd":1,"uuid":"d4555f9e-fc0e-4266-ae65-3c197d3fa02c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":29,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":782616861},{"type":"v1","addr":"172.21.15.152:6811","nonce":782616861}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":782616861},{"type":"v1","addr":"172.21.15.152:6813","nonce":782616861}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":782616861},{"type":"v1","addr":"172.21.15.152:6817","nonce":782616861}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":782616861},{"type":"v1","addr":"172.21.15.152:6815","nonce":782616861}]},"public_addr":"172.21.15.152:6811/782616861","cluster_addr":"172.21.15.152:6813/782616861","heartbeat_back_addr":"172.21.15.152:6817/782616861","heartbeat_front_addr":"172.21.15.152:6815/782616861","state":["exists","up"]},{"osd":2,"uuid":"caa0a912-a9d5-45e7-afe2-51beaf0f1742","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":1566571469},{"type":"v1","addr":"172.21.15.152:6819","nonce":1566571469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":1566571469},{"type":"v1","addr":"172.21.15.152:6821","nonce":1566571469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":1566571469},{"type":"v1","addr":"172.21.15.152:6825","nonce":1566571469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":1566571469},{"type":"v1","addr":"172.21.15.152:6823","nonce":1566571469}]},"public_addr":"172.21.15.152:6819/1566571469","cluster_addr":"172.21.15.152:6821/1566571469","heartbeat_back_addr":"172.21.15.152:6825/1566571469","heartbeat_front_addr":"172.21.15.152:6823/1566571469","state":["exists","up"]},{"osd":3,"uuid":"84a2d337-6ac8-41e4-a922-a6e94c1e9067","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6800","nonce":3366219775},{"type":"v1","addr":"172.21.15.154:6801","nonce":3366219775}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6802","nonce":3366219775},{"type":"v1","addr":"172.21.15.154:6803","nonce":3366219775}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6806","nonce":3366219775},{"type":"v1","addr":"172.21.15.154:6807","nonce":3366219775}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6804","nonce":3366219775},{"type":"v1","addr":"172.21.15.154:6805","nonce":3366219775}]},"public_addr":"172.21.15.154:6801/3366219775","cluster_addr":"172.21.15.154:6803/3366219775","heartbeat_back_addr":"172.21.15.154:6807/3366219775","heartbeat_front_addr":"172.21.15.154:6805/3366219775","state":["exists","up"]},{"osd":4,"uuid":"d547308c-898c-4165-9762-c8fd4170dc39","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":34,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6808","nonce":220818867},{"type":"v1","addr":"172.21.15.154:6809","nonce":220818867}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6810","nonce":220818867},{"type":"v1","addr":"172.21.15.154:6811","nonce":220818867}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6814","nonce":220818867},{"type":"v1","addr":"172.21.15.154:6815","nonce":220818867}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6812","nonce":220818867},{"type":"v1","addr":"172.21.15.154:6813","nonce":220818867}]},"public_addr":"172.21.15.154:6809/220818867","cluster_addr":"172.21.15.154:6811/220818867","heartbeat_back_addr":"172.21.15.154:6815/220818867","heartbeat_front_addr":"172.21.15.154:6813/220818867","state":["exists","up"]},{"osd":5,"uuid":"c3d3760a-08a7-44f2-8288-ccc5d08df057","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6816","nonce":1178927498},{"type":"v1","addr":"172.21.15.154:6817","nonce":1178927498}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6818","nonce":1178927498},{"type":"v1","addr":"172.21.15.154:6819","nonce":1178927498}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6822","nonce":1178927498},{"type":"v1","addr":"172.21.15.154:6823","nonce":1178927498}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6820","nonce":1178927498},{"type":"v1","addr":"172.21.15.154:6821","nonce":1178927498}]},"public_addr":"172.21.15.154:6817/1178927498","cluster_addr":"172.21.15.154:6819/1178927498","heartbeat_back_addr":"172.21.15.154:6823/1178927498","heartbeat_front_addr":"172.21.15.154:6821/1178927498","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:52:20.472492+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:52:49.979314+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:53:17.640952+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:53:40.392841+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:54:05.761710+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:54:30.235521+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.152:0/3027168373":"2024-04-03T04:51:09.949838+0000","172.21.15.152:0/2268166004":"2024-04-03T04:51:09.949838+0000","172.21.15.152:6800/2497421829":"2024-04-03T04:51:09.949838+0000","172.21.15.152:6801/2497421829":"2024-04-03T04:51:09.949838+0000","172.21.15.152:0/4188679877":"2024-04-03T04:50:01.902008+0000","172.21.15.152:0/1860988983":"2024-04-03T04:50:01.902008+0000","172.21.15.152:0/1943568671":"2024-04-03T04:49:30.819772+0000","172.21.15.152:0/2491428539":"2024-04-03T04:51:09.949838+0000","172.21.15.152:6801/3544077480":"2024-04-03T04:50:01.902008+0000","172.21.15.152:0/3476288520":"2024-04-03T04:49:30.819772+0000","172.21.15.152:6800/2913738627":"2024-04-03T04:49:30.819772+0000","172.21.15.152:6800/3544077480":"2024-04-03T04:50:01.902008+0000","172.21.15.152:0/3023512755":"2024-04-03T04:49:30.819772+0000","172.21.15.152:6801/2913738627":"2024-04-03T04:49:30.819772+0000","172.21.15.152:0/2947158817":"2024-04-03T04:50:01.902008+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-02T04:54:54.575 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.572+0000 7f3071fb6700 1 -- 172.21.15.152:0/1743190394 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f30580692a0 msgr2=0x7f305806b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:54.576 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.572+0000 7f3071fb6700 1 --2- 172.21.15.152:0/1743190394 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f30580692a0 0x7f305806b750 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 crypto rx=0x7f306c066b00 tx=0x7f3054008040 comp rx=0 tx=0).stop 2024-04-02T04:54:54.576 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.572+0000 7f3071fb6700 1 -- 172.21.15.152:0/1743190394 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f306c101d30 msgr2=0x7f306c10ec50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:54:54.576 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.572+0000 7f3071fb6700 1 --2- 172.21.15.152:0/1743190394 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f306c101d30 0x7f306c10ec50 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 crypto rx=0x7f306400be90 tx=0x7f3064004d10 comp rx=0 tx=0).stop 2024-04-02T04:54:54.576 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.572+0000 7f3071fb6700 1 -- 172.21.15.152:0/1743190394 shutdown_connections 2024-04-02T04:54:54.576 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.572+0000 7f3071fb6700 1 --2- 172.21.15.152:0/1743190394 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f30580692a0 0x7f305806b750 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:54.576 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.572+0000 7f3071fb6700 1 --2- 172.21.15.152:0/1743190394 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f306c101d30 0x7f306c10ec50 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:54.576 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.572+0000 7f3071fb6700 1 --2- 172.21.15.152:0/1743190394 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f306c1026d0 0x7f306c10f190 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:54:54.576 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.572+0000 7f3071fb6700 1 -- 172.21.15.152:0/1743190394 >> 172.21.15.152:0/1743190394 conn(0x7f306c0f78c0 msgr2=0x7f306c0f8850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:54:54.576 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.572+0000 7f3071fb6700 1 -- 172.21.15.152:0/1743190394 shutdown_connections 2024-04-02T04:54:54.576 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:54:54.572+0000 7f3071fb6700 1 -- 172.21.15.152:0/1743190394 wait complete. 2024-04-02T04:54:55.195 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:55 smithi152 ceph-mon[29073]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:54:55.195 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:55 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/1743190394' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T04:54:55.195 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:55 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:54:55.221 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph tell osd.0 flush_pg_stats 2024-04-02T04:54:55.221 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph tell osd.1 flush_pg_stats 2024-04-02T04:54:55.222 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph tell osd.2 flush_pg_stats 2024-04-02T04:54:55.222 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph tell osd.3 flush_pg_stats 2024-04-02T04:54:55.223 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph tell osd.4 flush_pg_stats 2024-04-02T04:54:55.223 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph tell osd.5 flush_pg_stats 2024-04-02T04:54:55.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:55 smithi154 ceph-mon[35729]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:54:55.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:55 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/1743190394' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T04:54:55.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:55 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:54:55.528 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:54:55.528 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:54:57.010 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:54:57.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:57 smithi152 ceph-mon[29073]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:54:57.386 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:54:57.386 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:54:57.432 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:54:57.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:57 smithi154 ceph-mon[35729]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:54:59.334 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:54:59 smithi152 ceph-mon[29073]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:54:59.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:54:59 smithi154 ceph-mon[35729]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:01.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:01 smithi152 ceph-mon[29073]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:01.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:01 smithi154 ceph-mon[35729]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:01.602 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.597+0000 7f72a7046700 1 -- 172.21.15.152:0/730510609 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f72a00fe7a0 msgr2=0x7f72a0106e70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:01.602 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.597+0000 7f72a7046700 1 --2- 172.21.15.152:0/730510609 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f72a00fe7a0 0x7f72a0106e70 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 crypto rx=0x7f7288004940 tx=0x7f7288009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:01.602 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.597+0000 7f72a7046700 1 -- 172.21.15.152:0/730510609 shutdown_connections 2024-04-02T04:55:01.602 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.597+0000 7f72a7046700 1 --2- 172.21.15.152:0/730510609 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f72a00fe7a0 0x7f72a0106e70 unknown :-1 s=CLOSED pgs=191 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:01.602 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.597+0000 7f72a7046700 1 --2- 172.21.15.152:0/730510609 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f72a00fde00 0x7f72a00fe1d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:01.602 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.597+0000 7f72a7046700 1 -- 172.21.15.152:0/730510609 >> 172.21.15.152:0/730510609 conn(0x7f72a00f7880 msgr2=0x7f72a00f9c90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:01.603 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.597+0000 7f72a7046700 1 -- 172.21.15.152:0/730510609 shutdown_connections 2024-04-02T04:55:01.603 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.597+0000 7f72a7046700 1 -- 172.21.15.152:0/730510609 wait complete. 2024-04-02T04:55:01.604 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.598+0000 7f72a7046700 1 Processor -- start 2024-04-02T04:55:01.604 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.598+0000 7f72a7046700 1 -- start start 2024-04-02T04:55:01.604 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.598+0000 7f72a7046700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f72a00fde00 0x7f72a0116090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:01.604 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.598+0000 7f72a7046700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f72a00fe7a0 0x7f72a01165d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:01.604 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.598+0000 7f72a7046700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f72a0116c20 con 0x7f72a00fe7a0 2024-04-02T04:55:01.604 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.598+0000 7f72a7046700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f72a010fff0 con 0x7f72a00fde00 2024-04-02T04:55:01.604 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.599+0000 7f729ffff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f72a00fe7a0 0x7f72a01165d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:01.604 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.599+0000 7f72a4de2700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f72a00fde00 0x7f72a0116090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:01.604 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.599+0000 7f729ffff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f72a00fe7a0 0x7f72a01165d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:39268/0 (socket says 172.21.15.152:39268) 2024-04-02T04:55:01.604 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.599+0000 7f729ffff700 1 -- 172.21.15.152:0/552903496 learned_addr learned my addr 172.21.15.152:0/552903496 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:55:01.604 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.599+0000 7f729ffff700 1 -- 172.21.15.152:0/552903496 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f72a00fde00 msgr2=0x7f72a0116090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:01.604 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.599+0000 7f729ffff700 1 --2- 172.21.15.152:0/552903496 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f72a00fde00 0x7f72a0116090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:01.604 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.599+0000 7f729ffff700 1 -- 172.21.15.152:0/552903496 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7288005040 con 0x7f72a00fe7a0 2024-04-02T04:55:01.605 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.599+0000 7f729ffff700 1 --2- 172.21.15.152:0/552903496 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f72a00fe7a0 0x7f72a01165d0 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f7288009380 tx=0x7f7288009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:01.605 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.599+0000 7f729d7fa700 1 -- 172.21.15.152:0/552903496 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f728800e040 con 0x7f72a00fe7a0 2024-04-02T04:55:01.605 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.599+0000 7f729d7fa700 1 -- 172.21.15.152:0/552903496 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f7288009af0 con 0x7f72a00fe7a0 2024-04-02T04:55:01.606 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.599+0000 7f72a7046700 1 -- 172.21.15.152:0/552903496 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f72a0110270 con 0x7f72a00fe7a0 2024-04-02T04:55:01.606 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.599+0000 7f729d7fa700 1 -- 172.21.15.152:0/552903496 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f728801c3f0 con 0x7f72a00fe7a0 2024-04-02T04:55:01.606 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.600+0000 7f72a7046700 1 -- 172.21.15.152:0/552903496 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f72a01107b0 con 0x7f72a00fe7a0 2024-04-02T04:55:01.606 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.600+0000 7f72a7046700 1 -- 172.21.15.152:0/552903496 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f72a00666f0 con 0x7f72a00fe7a0 2024-04-02T04:55:01.606 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.601+0000 7f729d7fa700 1 -- 172.21.15.152:0/552903496 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f728800baf0 con 0x7f72a00fe7a0 2024-04-02T04:55:01.607 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.601+0000 7f729d7fa700 1 --2- 172.21.15.152:0/552903496 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f728c0692a0 0x7f728c06b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:01.607 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.601+0000 7f729d7fa700 1 -- 172.21.15.152:0/552903496 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7f7288088190 con 0x7f72a00fe7a0 2024-04-02T04:55:01.607 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.601+0000 7f72a4de2700 1 --2- 172.21.15.152:0/552903496 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f728c0692a0 0x7f728c06b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:01.607 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.601+0000 7f729d7fa700 1 --2- 172.21.15.152:0/552903496 >> [v2:172.21.15.154:6800/3366219775,v1:172.21.15.154:6801/3366219775] conn(0x7f728c06ef70 0x7f728c071380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:01.607 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.602+0000 7f729d7fa700 1 -- 172.21.15.152:0/552903496 --> [v2:172.21.15.154:6800/3366219775,v1:172.21.15.154:6801/3366219775] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f728c071a30 con 0x7f728c06ef70 2024-04-02T04:55:01.607 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.602+0000 7f729d7fa700 1 -- 172.21.15.152:0/552903496 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_get_version_reply(handle=1 version=40) v2 ==== 24+0+0 (secure 0 0 0) 0x7f728808a780 con 0x7f72a00fe7a0 2024-04-02T04:55:01.607 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.602+0000 7f72a55e3700 1 --2- 172.21.15.152:0/552903496 >> [v2:172.21.15.154:6800/3366219775,v1:172.21.15.154:6801/3366219775] conn(0x7f728c06ef70 0x7f728c071380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:01.607 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.602+0000 7f72a4de2700 1 --2- 172.21.15.152:0/552903496 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f728c0692a0 0x7f728c06b750 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f7298007a40 tx=0x7f7298009da0 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:01.607 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.602+0000 7f72a55e3700 1 --2- 172.21.15.152:0/552903496 >> [v2:172.21.15.154:6800/3366219775,v1:172.21.15.154:6801/3366219775] conn(0x7f728c06ef70 0x7f728c071380 crc :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.3 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:01.608 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.603+0000 7f729d7fa700 1 -- 172.21.15.152:0/552903496 <== osd.3 v2:172.21.15.154:6800/3366219775 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+24147 (crc 0 0 0) 0x7f728c071a30 con 0x7f728c06ef70 2024-04-02T04:55:01.622 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.618+0000 7f72a7046700 1 -- 172.21.15.152:0/552903496 --> [v2:172.21.15.154:6800/3366219775,v1:172.21.15.154:6801/3366219775] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f72a0110d50 con 0x7f728c06ef70 2024-04-02T04:55:01.623 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.619+0000 7f729d7fa700 1 -- 172.21.15.152:0/552903496 <== osd.3 v2:172.21.15.154:6800/3366219775 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+12 (crc 0 0 0) 0x7f72a0110d50 con 0x7f728c06ef70 2024-04-02T04:55:01.624 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.619+0000 7f72a7046700 1 -- 172.21.15.152:0/552903496 >> [v2:172.21.15.154:6800/3366219775,v1:172.21.15.154:6801/3366219775] conn(0x7f728c06ef70 msgr2=0x7f728c071380 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:01.624 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.619+0000 7f72a7046700 1 --2- 172.21.15.152:0/552903496 >> [v2:172.21.15.154:6800/3366219775,v1:172.21.15.154:6801/3366219775] conn(0x7f728c06ef70 0x7f728c071380 crc :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:01.624 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.619+0000 7f72a7046700 1 -- 172.21.15.152:0/552903496 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f728c0692a0 msgr2=0x7f728c06b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:01.624 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.619+0000 7f72a7046700 1 --2- 172.21.15.152:0/552903496 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f728c0692a0 0x7f728c06b750 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 crypto rx=0x7f7298007a40 tx=0x7f7298009da0 comp rx=0 tx=0).stop 2024-04-02T04:55:01.624 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.619+0000 7f72a7046700 1 -- 172.21.15.152:0/552903496 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f72a00fe7a0 msgr2=0x7f72a01165d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:01.624 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.619+0000 7f72a7046700 1 --2- 172.21.15.152:0/552903496 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f72a00fe7a0 0x7f72a01165d0 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 crypto rx=0x7f7288009380 tx=0x7f7288009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:01.624 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.619+0000 7f72a7046700 1 -- 172.21.15.152:0/552903496 shutdown_connections 2024-04-02T04:55:01.625 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.619+0000 7f72a7046700 1 --2- 172.21.15.152:0/552903496 >> [v2:172.21.15.154:6800/3366219775,v1:172.21.15.154:6801/3366219775] conn(0x7f728c06ef70 0x7f728c071380 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:01.625 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.619+0000 7f72a7046700 1 --2- 172.21.15.152:0/552903496 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f728c0692a0 0x7f728c06b750 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:01.625 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.619+0000 7f72a7046700 1 --2- 172.21.15.152:0/552903496 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f72a00fde00 0x7f72a0116090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:01.625 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.619+0000 7f72a7046700 1 --2- 172.21.15.152:0/552903496 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f72a00fe7a0 0x7f72a01165d0 unknown :-1 s=CLOSED pgs=192 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:01.625 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.619+0000 7f72a7046700 1 -- 172.21.15.152:0/552903496 >> 172.21.15.152:0/552903496 conn(0x7f72a00f7880 msgr2=0x7f72a00f8d50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:01.625 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.619+0000 7f72a7046700 1 -- 172.21.15.152:0/552903496 shutdown_connections 2024-04-02T04:55:01.625 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.619+0000 7f72a7046700 1 -- 172.21.15.152:0/552903496 wait complete. 2024-04-02T04:55:01.880 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.874+0000 7f4cff947700 1 -- 172.21.15.152:0/644481490 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4cf8102e00 msgr2=0x7f4cf81031d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:01.881 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.874+0000 7f4cff947700 1 --2- 172.21.15.152:0/644481490 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4cf8102e00 0x7f4cf81031d0 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 crypto rx=0x7f4cf4004940 tx=0x7f4cf4009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:01.881 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.874+0000 7f4cff947700 1 -- 172.21.15.152:0/644481490 shutdown_connections 2024-04-02T04:55:01.881 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.874+0000 7f4cff947700 1 --2- 172.21.15.152:0/644481490 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4cf81037a0 0x7f4cf80fce10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:01.881 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.874+0000 7f4cff947700 1 --2- 172.21.15.152:0/644481490 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4cf8102e00 0x7f4cf81031d0 unknown :-1 s=CLOSED pgs=193 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:01.881 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.874+0000 7f4cff947700 1 -- 172.21.15.152:0/644481490 >> 172.21.15.152:0/644481490 conn(0x7f4cf80f89b0 msgr2=0x7f4cf80fadc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:01.881 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.874+0000 7f4cff947700 1 -- 172.21.15.152:0/644481490 shutdown_connections 2024-04-02T04:55:01.881 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.875+0000 7f4cff947700 1 -- 172.21.15.152:0/644481490 wait complete. 2024-04-02T04:55:01.881 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.875+0000 7f4cff947700 1 Processor -- start 2024-04-02T04:55:01.881 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.875+0000 7f4cff947700 1 -- start start 2024-04-02T04:55:01.881 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.875+0000 7f4cff947700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4cf8102e00 0x7f4cf80702c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:01.881 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.875+0000 7f4cff947700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4cf81037a0 0x7f4cf8070800 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:01.881 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.875+0000 7f4cff947700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4cf80721c0 con 0x7f4cf8102e00 2024-04-02T04:55:01.881 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.875+0000 7f4cff947700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4cf8070d40 con 0x7f4cf81037a0 2024-04-02T04:55:01.882 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.875+0000 7f4cfd6e3700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4cf8102e00 0x7f4cf80702c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:01.882 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.875+0000 7f4cfd6e3700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4cf8102e00 0x7f4cf80702c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:39284/0 (socket says 172.21.15.152:39284) 2024-04-02T04:55:01.882 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.875+0000 7f4cfd6e3700 1 -- 172.21.15.152:0/835316257 learned_addr learned my addr 172.21.15.152:0/835316257 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:55:01.882 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.875+0000 7f4cfcee2700 1 --2- 172.21.15.152:0/835316257 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4cf81037a0 0x7f4cf8070800 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:01.882 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.876+0000 7f4cfd6e3700 1 -- 172.21.15.152:0/835316257 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4cf81037a0 msgr2=0x7f4cf8070800 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:01.882 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.876+0000 7f4cfd6e3700 1 --2- 172.21.15.152:0/835316257 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4cf81037a0 0x7f4cf8070800 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:01.882 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.876+0000 7f4cfd6e3700 1 -- 172.21.15.152:0/835316257 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4cf4005040 con 0x7f4cf8102e00 2024-04-02T04:55:01.882 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.876+0000 7f4cfd6e3700 1 --2- 172.21.15.152:0/835316257 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4cf8102e00 0x7f4cf80702c0 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f4cf400b3b0 tx=0x7f4cf4009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:01.882 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.876+0000 7f4cedffb700 1 -- 172.21.15.152:0/835316257 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f4cf400e040 con 0x7f4cf8102e00 2024-04-02T04:55:01.882 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.876+0000 7f4cedffb700 1 -- 172.21.15.152:0/835316257 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f4cf400bbc0 con 0x7f4cf8102e00 2024-04-02T04:55:01.882 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.876+0000 7f4cff947700 1 -- 172.21.15.152:0/835316257 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4cf8070fc0 con 0x7f4cf8102e00 2024-04-02T04:55:01.882 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.876+0000 7f4cedffb700 1 -- 172.21.15.152:0/835316257 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f4cf401c3f0 con 0x7f4cf8102e00 2024-04-02T04:55:01.884 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.876+0000 7f4cff947700 1 -- 172.21.15.152:0/835316257 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4cf811a130 con 0x7f4cf8102e00 2024-04-02T04:55:01.884 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.877+0000 7f4cff947700 1 -- 172.21.15.152:0/835316257 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f4cf80fe550 con 0x7f4cf8102e00 2024-04-02T04:55:01.884 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.877+0000 7f4cedffb700 1 -- 172.21.15.152:0/835316257 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f4cf400bd30 con 0x7f4cf8102e00 2024-04-02T04:55:01.884 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.878+0000 7f4cedffb700 1 --2- 172.21.15.152:0/835316257 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f4ce40692a0 0x7f4ce406b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:01.884 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.878+0000 7f4cedffb700 1 -- 172.21.15.152:0/835316257 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7f4cf40880e0 con 0x7f4cf8102e00 2024-04-02T04:55:01.884 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.878+0000 7f4cfcee2700 1 --2- 172.21.15.152:0/835316257 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f4ce40692a0 0x7f4ce406b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:01.884 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.878+0000 7f4cedffb700 1 --2- 172.21.15.152:0/835316257 >> [v2:172.21.15.152:6818/1566571469,v1:172.21.15.152:6819/1566571469] conn(0x7f4ce406ef70 0x7f4ce4071380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:01.884 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.878+0000 7f4cedffb700 1 -- 172.21.15.152:0/835316257 --> [v2:172.21.15.152:6818/1566571469,v1:172.21.15.152:6819/1566571469] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f4ce4071a30 con 0x7f4ce406ef70 2024-04-02T04:55:01.884 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.878+0000 7f4cedffb700 1 -- 172.21.15.152:0/835316257 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_get_version_reply(handle=1 version=40) v2 ==== 24+0+0 (secure 0 0 0) 0x7f4cf408a6d0 con 0x7f4cf8102e00 2024-04-02T04:55:01.884 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.878+0000 7f4cfdee4700 1 --2- 172.21.15.152:0/835316257 >> [v2:172.21.15.152:6818/1566571469,v1:172.21.15.152:6819/1566571469] conn(0x7f4ce406ef70 0x7f4ce4071380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:01.885 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.878+0000 7f4cfcee2700 1 --2- 172.21.15.152:0/835316257 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f4ce40692a0 0x7f4ce406b750 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f4ce8007fd0 tx=0x7f4ce8009040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:01.885 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.879+0000 7f4cfdee4700 1 --2- 172.21.15.152:0/835316257 >> [v2:172.21.15.152:6818/1566571469,v1:172.21.15.152:6819/1566571469] conn(0x7f4ce406ef70 0x7f4ce4071380 crc :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.2 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:01.885 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.879+0000 7f4cedffb700 1 -- 172.21.15.152:0/835316257 <== osd.2 v2:172.21.15.152:6818/1566571469 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+24147 (crc 0 0 0) 0x7f4ce4071a30 con 0x7f4ce406ef70 2024-04-02T04:55:01.898 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.894+0000 7f4cff947700 1 -- 172.21.15.152:0/835316257 --> [v2:172.21.15.152:6818/1566571469,v1:172.21.15.152:6819/1566571469] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f4cf805e640 con 0x7f4ce406ef70 2024-04-02T04:55:01.899 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.894+0000 7f4cedffb700 1 -- 172.21.15.152:0/835316257 <== osd.2 v2:172.21.15.152:6818/1566571469 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (crc 0 0 0) 0x7f4cf805e640 con 0x7f4ce406ef70 2024-04-02T04:55:01.900 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.895+0000 7f4cff947700 1 -- 172.21.15.152:0/835316257 >> [v2:172.21.15.152:6818/1566571469,v1:172.21.15.152:6819/1566571469] conn(0x7f4ce406ef70 msgr2=0x7f4ce4071380 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:01.900 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.895+0000 7f4cff947700 1 --2- 172.21.15.152:0/835316257 >> [v2:172.21.15.152:6818/1566571469,v1:172.21.15.152:6819/1566571469] conn(0x7f4ce406ef70 0x7f4ce4071380 crc :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:01.900 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.895+0000 7f4cff947700 1 -- 172.21.15.152:0/835316257 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f4ce40692a0 msgr2=0x7f4ce406b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:01.900 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.895+0000 7f4cff947700 1 --2- 172.21.15.152:0/835316257 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f4ce40692a0 0x7f4ce406b750 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 crypto rx=0x7f4ce8007fd0 tx=0x7f4ce8009040 comp rx=0 tx=0).stop 2024-04-02T04:55:01.900 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.895+0000 7f4cff947700 1 -- 172.21.15.152:0/835316257 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4cf8102e00 msgr2=0x7f4cf80702c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:01.900 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.895+0000 7f4cff947700 1 --2- 172.21.15.152:0/835316257 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4cf8102e00 0x7f4cf80702c0 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 crypto rx=0x7f4cf400b3b0 tx=0x7f4cf4009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:01.900 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.895+0000 7f4cff947700 1 -- 172.21.15.152:0/835316257 shutdown_connections 2024-04-02T04:55:01.901 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.895+0000 7f4cff947700 1 --2- 172.21.15.152:0/835316257 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f4ce40692a0 0x7f4ce406b750 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:01.901 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.895+0000 7f4cff947700 1 --2- 172.21.15.152:0/835316257 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4cf8102e00 0x7f4cf80702c0 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:01.901 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.895+0000 7f4cff947700 1 --2- 172.21.15.152:0/835316257 >> [v2:172.21.15.152:6818/1566571469,v1:172.21.15.152:6819/1566571469] conn(0x7f4ce406ef70 0x7f4ce4071380 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:01.901 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.895+0000 7f4cff947700 1 --2- 172.21.15.152:0/835316257 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4cf81037a0 0x7f4cf8070800 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:01.901 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.895+0000 7f4cff947700 1 -- 172.21.15.152:0/835316257 >> 172.21.15.152:0/835316257 conn(0x7f4cf80f89b0 msgr2=0x7f4cf81073a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:01.901 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.895+0000 7f4cff947700 1 -- 172.21.15.152:0/835316257 shutdown_connections 2024-04-02T04:55:01.901 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:01.895+0000 7f4cff947700 1 -- 172.21.15.152:0/835316257 wait complete. 2024-04-02T04:55:02.558 INFO:teuthology.orchestra.run.smithi152.stdout:111669149714 2024-04-02T04:55:02.559 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.3 2024-04-02T04:55:03.286 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:03 smithi152 ceph-mon[29073]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:03.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:03 smithi154 ceph-mon[35729]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:04.111 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.105+0000 7f4a608c3700 1 -- 172.21.15.152:0/451790473 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4a5c0fde30 msgr2=0x7f4a5c0fe200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:04.112 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.105+0000 7f4a608c3700 1 --2- 172.21.15.152:0/451790473 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4a5c0fde30 0x7f4a5c0fe200 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 crypto rx=0x7f4a4c004940 tx=0x7f4a4c009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:04.112 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.105+0000 7f4a608c3700 1 -- 172.21.15.152:0/451790473 shutdown_connections 2024-04-02T04:55:04.112 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.105+0000 7f4a608c3700 1 --2- 172.21.15.152:0/451790473 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4a5c0fe7d0 0x7f4a5c106e10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:04.112 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.105+0000 7f4a608c3700 1 --2- 172.21.15.152:0/451790473 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4a5c0fde30 0x7f4a5c0fe200 unknown :-1 s=CLOSED pgs=195 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:04.112 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.105+0000 7f4a608c3700 1 -- 172.21.15.152:0/451790473 >> 172.21.15.152:0/451790473 conn(0x7f4a5c0f78c0 msgr2=0x7f4a5c0f9cd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:04.112 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.105+0000 7f4a608c3700 1 -- 172.21.15.152:0/451790473 shutdown_connections 2024-04-02T04:55:04.112 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.105+0000 7f4a608c3700 1 -- 172.21.15.152:0/451790473 wait complete. 2024-04-02T04:55:04.112 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.105+0000 7f4a608c3700 1 Processor -- start 2024-04-02T04:55:04.112 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.106+0000 7f4a608c3700 1 -- start start 2024-04-02T04:55:04.112 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.106+0000 7f4a608c3700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4a5c0fde30 0x7f4a5c113a40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:04.112 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.106+0000 7f4a608c3700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4a5c0fe7d0 0x7f4a5c113f80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:04.113 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.106+0000 7f4a608c3700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4a5c1145d0 con 0x7f4a5c0fe7d0 2024-04-02T04:55:04.113 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.106+0000 7f4a608c3700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4a5c10dbb0 con 0x7f4a5c0fde30 2024-04-02T04:55:04.113 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.106+0000 7f4a59d9b700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4a5c0fe7d0 0x7f4a5c113f80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:04.113 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.106+0000 7f4a59d9b700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4a5c0fe7d0 0x7f4a5c113f80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:39302/0 (socket says 172.21.15.152:39302) 2024-04-02T04:55:04.113 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.106+0000 7f4a59d9b700 1 -- 172.21.15.152:0/2920058587 learned_addr learned my addr 172.21.15.152:0/2920058587 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:55:04.113 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.106+0000 7f4a5a59c700 1 --2- 172.21.15.152:0/2920058587 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4a5c0fde30 0x7f4a5c113a40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:04.113 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.106+0000 7f4a59d9b700 1 -- 172.21.15.152:0/2920058587 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4a5c0fde30 msgr2=0x7f4a5c113a40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:04.113 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.106+0000 7f4a59d9b700 1 --2- 172.21.15.152:0/2920058587 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4a5c0fde30 0x7f4a5c113a40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:04.113 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.106+0000 7f4a59d9b700 1 -- 172.21.15.152:0/2920058587 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4a4c005040 con 0x7f4a5c0fe7d0 2024-04-02T04:55:04.113 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.106+0000 7f4a59d9b700 1 --2- 172.21.15.152:0/2920058587 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4a5c0fe7d0 0x7f4a5c113f80 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f4a50002a50 tx=0x7f4a500046b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:04.113 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.107+0000 7f4a4affd700 1 -- 172.21.15.152:0/2920058587 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f4a50007cc0 con 0x7f4a5c0fe7d0 2024-04-02T04:55:04.113 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.107+0000 7f4a608c3700 1 -- 172.21.15.152:0/2920058587 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4a5c10de90 con 0x7f4a5c0fe7d0 2024-04-02T04:55:04.114 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.107+0000 7f4a608c3700 1 -- 172.21.15.152:0/2920058587 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4a5c10e3d0 con 0x7f4a5c0fe7d0 2024-04-02T04:55:04.114 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.107+0000 7f4a4affd700 1 -- 172.21.15.152:0/2920058587 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f4a5000f040 con 0x7f4a5c0fe7d0 2024-04-02T04:55:04.114 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.107+0000 7f4a4affd700 1 -- 172.21.15.152:0/2920058587 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f4a5000da30 con 0x7f4a5c0fe7d0 2024-04-02T04:55:04.115 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.107+0000 7f4a608c3700 1 -- 172.21.15.152:0/2920058587 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f4a3c000fd0 con 0x7f4a5c0fe7d0 2024-04-02T04:55:04.115 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.108+0000 7f4a4affd700 1 -- 172.21.15.152:0/2920058587 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f4a5000db90 con 0x7f4a5c0fe7d0 2024-04-02T04:55:04.115 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.108+0000 7f4a4affd700 1 --2- 172.21.15.152:0/2920058587 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f4a440692a0 0x7f4a4406b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:04.115 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.109+0000 7f4a4affd700 1 -- 172.21.15.152:0/2920058587 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7f4a50087510 con 0x7f4a5c0fe7d0 2024-04-02T04:55:04.115 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.109+0000 7f4a5a59c700 1 --2- 172.21.15.152:0/2920058587 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f4a440692a0 0x7f4a4406b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:04.115 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.109+0000 7f4a4affd700 1 --2- 172.21.15.152:0/2920058587 >> [v2:172.21.15.154:6816/1178927498,v1:172.21.15.154:6817/1178927498] conn(0x7f4a4406ef70 0x7f4a44071380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:04.115 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.109+0000 7f4a4affd700 1 -- 172.21.15.152:0/2920058587 --> [v2:172.21.15.154:6816/1178927498,v1:172.21.15.154:6817/1178927498] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f4a44071a30 con 0x7f4a4406ef70 2024-04-02T04:55:04.116 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.109+0000 7f4a4affd700 1 -- 172.21.15.152:0/2920058587 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_get_version_reply(handle=1 version=40) v2 ==== 24+0+0 (secure 0 0 0) 0x7f4a500878f0 con 0x7f4a5c0fe7d0 2024-04-02T04:55:04.116 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.109+0000 7f4a5a59c700 1 --2- 172.21.15.152:0/2920058587 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f4a440692a0 0x7f4a4406b750 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f4a4c004f90 tx=0x7f4a4c009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:04.116 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.109+0000 7f4a5ad9d700 1 --2- 172.21.15.152:0/2920058587 >> [v2:172.21.15.154:6816/1178927498,v1:172.21.15.154:6817/1178927498] conn(0x7f4a4406ef70 0x7f4a44071380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:04.116 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.109+0000 7f4a5ad9d700 1 --2- 172.21.15.152:0/2920058587 >> [v2:172.21.15.154:6816/1178927498,v1:172.21.15.154:6817/1178927498] conn(0x7f4a4406ef70 0x7f4a44071380 crc :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.5 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:04.116 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.110+0000 7f4a4affd700 1 -- 172.21.15.152:0/2920058587 <== osd.5 v2:172.21.15.154:6816/1178927498 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+24147 (crc 0 0 0) 0x7f4a44071a30 con 0x7f4a4406ef70 2024-04-02T04:55:04.130 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.126+0000 7f4a608c3700 1 -- 172.21.15.152:0/2920058587 --> [v2:172.21.15.154:6816/1178927498,v1:172.21.15.154:6817/1178927498] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f4a3c002d30 con 0x7f4a4406ef70 2024-04-02T04:55:04.131 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.126+0000 7f4a4affd700 1 -- 172.21.15.152:0/2920058587 <== osd.5 v2:172.21.15.154:6816/1178927498 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+12 (crc 0 0 0) 0x7f4a3c002d30 con 0x7f4a4406ef70 2024-04-02T04:55:04.132 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.127+0000 7f4a608c3700 1 -- 172.21.15.152:0/2920058587 >> [v2:172.21.15.154:6816/1178927498,v1:172.21.15.154:6817/1178927498] conn(0x7f4a4406ef70 msgr2=0x7f4a44071380 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:04.132 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.127+0000 7f4a608c3700 1 --2- 172.21.15.152:0/2920058587 >> [v2:172.21.15.154:6816/1178927498,v1:172.21.15.154:6817/1178927498] conn(0x7f4a4406ef70 0x7f4a44071380 crc :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:04.132 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.127+0000 7f4a608c3700 1 -- 172.21.15.152:0/2920058587 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f4a440692a0 msgr2=0x7f4a4406b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:04.132 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.127+0000 7f4a608c3700 1 --2- 172.21.15.152:0/2920058587 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f4a440692a0 0x7f4a4406b750 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 crypto rx=0x7f4a4c004f90 tx=0x7f4a4c009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:04.133 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.127+0000 7f4a608c3700 1 -- 172.21.15.152:0/2920058587 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4a5c0fe7d0 msgr2=0x7f4a5c113f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:04.133 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.127+0000 7f4a608c3700 1 --2- 172.21.15.152:0/2920058587 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4a5c0fe7d0 0x7f4a5c113f80 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 crypto rx=0x7f4a50002a50 tx=0x7f4a500046b0 comp rx=0 tx=0).stop 2024-04-02T04:55:04.133 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.127+0000 7f4a608c3700 1 -- 172.21.15.152:0/2920058587 shutdown_connections 2024-04-02T04:55:04.133 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.127+0000 7f4a608c3700 1 --2- 172.21.15.152:0/2920058587 >> [v2:172.21.15.154:6816/1178927498,v1:172.21.15.154:6817/1178927498] conn(0x7f4a4406ef70 0x7f4a44071380 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:04.133 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.127+0000 7f4a608c3700 1 --2- 172.21.15.152:0/2920058587 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f4a440692a0 0x7f4a4406b750 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:04.133 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.127+0000 7f4a608c3700 1 --2- 172.21.15.152:0/2920058587 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4a5c0fde30 0x7f4a5c113a40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:04.133 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.127+0000 7f4a608c3700 1 --2- 172.21.15.152:0/2920058587 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4a5c0fe7d0 0x7f4a5c113f80 unknown :-1 s=CLOSED pgs=196 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:04.133 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.127+0000 7f4a608c3700 1 -- 172.21.15.152:0/2920058587 >> 172.21.15.152:0/2920058587 conn(0x7f4a5c0f78c0 msgr2=0x7f4a5c0f8ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:04.133 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.127+0000 7f4a608c3700 1 -- 172.21.15.152:0/2920058587 shutdown_connections 2024-04-02T04:55:04.133 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:04.127+0000 7f4a608c3700 1 -- 172.21.15.152:0/2920058587 wait complete. 2024-04-02T04:55:05.083 INFO:teuthology.orchestra.run.smithi152.stdout:81604378648 2024-04-02T04:55:05.084 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.2 2024-04-02T04:55:05.328 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:05 smithi152 ceph-mon[29073]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:05.428 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.421+0000 7fd671a7a700 1 -- 172.21.15.152:0/2828400434 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd66c102710 msgr2=0x7fd66c0fbd80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:05.429 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.421+0000 7fd671a7a700 1 --2- 172.21.15.152:0/2828400434 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd66c102710 0x7fd66c0fbd80 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 crypto rx=0x7fd660004940 tx=0x7fd660009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:05.429 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.421+0000 7fd671a7a700 1 -- 172.21.15.152:0/2828400434 shutdown_connections 2024-04-02T04:55:05.429 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.421+0000 7fd671a7a700 1 --2- 172.21.15.152:0/2828400434 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd66c102710 0x7fd66c0fbd80 unknown :-1 s=CLOSED pgs=197 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:05.429 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.421+0000 7fd671a7a700 1 --2- 172.21.15.152:0/2828400434 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd66c101d70 0x7fd66c102140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:05.430 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.421+0000 7fd671a7a700 1 -- 172.21.15.152:0/2828400434 >> 172.21.15.152:0/2828400434 conn(0x7fd66c072bc0 msgr2=0x7fd66c072fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:05.430 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.421+0000 7fd671a7a700 1 -- 172.21.15.152:0/2828400434 shutdown_connections 2024-04-02T04:55:05.430 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.421+0000 7fd671a7a700 1 -- 172.21.15.152:0/2828400434 wait complete. 2024-04-02T04:55:05.430 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.422+0000 7fd671a7a700 1 Processor -- start 2024-04-02T04:55:05.430 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.422+0000 7fd671a7a700 1 -- start start 2024-04-02T04:55:05.430 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.422+0000 7fd671a7a700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd66c101d70 0x7fd66c1129e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:05.430 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.422+0000 7fd671a7a700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd66c102710 0x7fd66c112f20 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:05.430 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.422+0000 7fd671a7a700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd66c113570 con 0x7fd66c102710 2024-04-02T04:55:05.430 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.422+0000 7fd671a7a700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd66c10c960 con 0x7fd66c101d70 2024-04-02T04:55:05.430 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.422+0000 7fd66affd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd66c102710 0x7fd66c112f20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:05.430 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.422+0000 7fd66affd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd66c102710 0x7fd66c112f20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:39314/0 (socket says 172.21.15.152:39314) 2024-04-02T04:55:05.431 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.422+0000 7fd66affd700 1 -- 172.21.15.152:0/347630252 learned_addr learned my addr 172.21.15.152:0/347630252 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:55:05.431 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.422+0000 7fd66b7fe700 1 --2- 172.21.15.152:0/347630252 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd66c101d70 0x7fd66c1129e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:05.431 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.423+0000 7fd66affd700 1 -- 172.21.15.152:0/347630252 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd66c101d70 msgr2=0x7fd66c1129e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:05.431 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.423+0000 7fd66affd700 1 --2- 172.21.15.152:0/347630252 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd66c101d70 0x7fd66c1129e0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:05.431 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.423+0000 7fd66affd700 1 -- 172.21.15.152:0/347630252 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd660005040 con 0x7fd66c102710 2024-04-02T04:55:05.431 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.423+0000 7fd66affd700 1 --2- 172.21.15.152:0/347630252 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd66c102710 0x7fd66c112f20 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7fd660009180 tx=0x7fd660009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:05.431 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.423+0000 7fd653fff700 1 -- 172.21.15.152:0/347630252 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fd66000e040 con 0x7fd66c102710 2024-04-02T04:55:05.431 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.423+0000 7fd671a7a700 1 -- 172.21.15.152:0/347630252 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd66c10cbe0 con 0x7fd66c102710 2024-04-02T04:55:05.431 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.423+0000 7fd653fff700 1 -- 172.21.15.152:0/347630252 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fd660013420 con 0x7fd66c102710 2024-04-02T04:55:05.431 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.423+0000 7fd653fff700 1 -- 172.21.15.152:0/347630252 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fd66001c610 con 0x7fd66c102710 2024-04-02T04:55:05.431 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.423+0000 7fd671a7a700 1 -- 172.21.15.152:0/347630252 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd66c10d1a0 con 0x7fd66c102710 2024-04-02T04:55:05.431 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.424+0000 7fd671a7a700 1 -- 172.21.15.152:0/347630252 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7fd66c10d790 con 0x7fd66c102710 2024-04-02T04:55:05.433 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.425+0000 7fd653fff700 1 -- 172.21.15.152:0/347630252 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7fd660013a00 con 0x7fd66c102710 2024-04-02T04:55:05.433 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.425+0000 7fd653fff700 1 --2- 172.21.15.152:0/347630252 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd6540692a0 0x7fd65406b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:05.433 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.425+0000 7fd653fff700 1 -- 172.21.15.152:0/347630252 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7fd660087f50 con 0x7fd66c102710 2024-04-02T04:55:05.433 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.425+0000 7fd66b7fe700 1 --2- 172.21.15.152:0/347630252 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd6540692a0 0x7fd65406b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:05.433 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.425+0000 7fd653fff700 1 --2- 172.21.15.152:0/347630252 >> [v2:172.21.15.152:6802/718734380,v1:172.21.15.152:6803/718734380] conn(0x7fd65406ef70 0x7fd654071380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:05.433 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.425+0000 7fd653fff700 1 -- 172.21.15.152:0/347630252 --> [v2:172.21.15.152:6802/718734380,v1:172.21.15.152:6803/718734380] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7fd654071a30 con 0x7fd65406ef70 2024-04-02T04:55:05.433 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.425+0000 7fd653fff700 1 -- 172.21.15.152:0/347630252 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_get_version_reply(handle=1 version=40) v2 ==== 24+0+0 (secure 0 0 0) 0x7fd66008a540 con 0x7fd66c102710 2024-04-02T04:55:05.433 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.425+0000 7fd66bfff700 1 --2- 172.21.15.152:0/347630252 >> [v2:172.21.15.152:6802/718734380,v1:172.21.15.152:6803/718734380] conn(0x7fd65406ef70 0x7fd654071380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:05.433 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.425+0000 7fd66b7fe700 1 --2- 172.21.15.152:0/347630252 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd6540692a0 0x7fd65406b750 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7fd65c009fd0 tx=0x7fd65c00b040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:05.433 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.426+0000 7fd66bfff700 1 --2- 172.21.15.152:0/347630252 >> [v2:172.21.15.152:6802/718734380,v1:172.21.15.152:6803/718734380] conn(0x7fd65406ef70 0x7fd654071380 crc :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:05.433 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.426+0000 7fd653fff700 1 -- 172.21.15.152:0/347630252 <== osd.0 v2:172.21.15.152:6802/718734380 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+24147 (crc 0 0 0) 0x7fd654071a30 con 0x7fd65406ef70 2024-04-02T04:55:05.446 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.442+0000 7fd671a7a700 1 -- 172.21.15.152:0/347630252 --> [v2:172.21.15.152:6802/718734380,v1:172.21.15.152:6803/718734380] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7fd66c05e640 con 0x7fd65406ef70 2024-04-02T04:55:05.447 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.442+0000 7fd653fff700 1 -- 172.21.15.152:0/347630252 <== osd.0 v2:172.21.15.152:6802/718734380 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (crc 0 0 0) 0x7fd66c05e640 con 0x7fd65406ef70 2024-04-02T04:55:05.448 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.442+0000 7fd671a7a700 1 -- 172.21.15.152:0/347630252 >> [v2:172.21.15.152:6802/718734380,v1:172.21.15.152:6803/718734380] conn(0x7fd65406ef70 msgr2=0x7fd654071380 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:05.448 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.442+0000 7fd671a7a700 1 --2- 172.21.15.152:0/347630252 >> [v2:172.21.15.152:6802/718734380,v1:172.21.15.152:6803/718734380] conn(0x7fd65406ef70 0x7fd654071380 crc :-1 s=READY pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:05.448 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.442+0000 7fd671a7a700 1 -- 172.21.15.152:0/347630252 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd6540692a0 msgr2=0x7fd65406b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:05.448 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.442+0000 7fd671a7a700 1 --2- 172.21.15.152:0/347630252 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd6540692a0 0x7fd65406b750 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 crypto rx=0x7fd65c009fd0 tx=0x7fd65c00b040 comp rx=0 tx=0).stop 2024-04-02T04:55:05.448 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.442+0000 7fd671a7a700 1 -- 172.21.15.152:0/347630252 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd66c102710 msgr2=0x7fd66c112f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:05.448 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.442+0000 7fd671a7a700 1 --2- 172.21.15.152:0/347630252 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd66c102710 0x7fd66c112f20 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 crypto rx=0x7fd660009180 tx=0x7fd660009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:05.448 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.443+0000 7fd671a7a700 1 -- 172.21.15.152:0/347630252 shutdown_connections 2024-04-02T04:55:05.448 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.443+0000 7fd671a7a700 1 --2- 172.21.15.152:0/347630252 >> [v2:172.21.15.152:6802/718734380,v1:172.21.15.152:6803/718734380] conn(0x7fd65406ef70 0x7fd654071380 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:05.448 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.443+0000 7fd671a7a700 1 --2- 172.21.15.152:0/347630252 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd6540692a0 0x7fd65406b750 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:05.448 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.443+0000 7fd671a7a700 1 --2- 172.21.15.152:0/347630252 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd66c101d70 0x7fd66c1129e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:05.448 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.443+0000 7fd671a7a700 1 --2- 172.21.15.152:0/347630252 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd66c102710 0x7fd66c112f20 unknown :-1 s=CLOSED pgs=198 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:05.449 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.443+0000 7fd671a7a700 1 -- 172.21.15.152:0/347630252 >> 172.21.15.152:0/347630252 conn(0x7fd66c072bc0 msgr2=0x7fd66c0faf20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:05.449 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.443+0000 7fd671a7a700 1 -- 172.21.15.152:0/347630252 shutdown_connections 2024-04-02T04:55:05.449 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.443+0000 7fd671a7a700 1 -- 172.21.15.152:0/347630252 wait complete. 2024-04-02T04:55:05.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:05 smithi154 ceph-mon[35729]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:05.670 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.640+0000 7fb6b231e700 1 -- 172.21.15.152:0/2942839204 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb6ac101d70 msgr2=0x7fb6ac102140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:05.671 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.640+0000 7fb6b231e700 1 --2- 172.21.15.152:0/2942839204 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb6ac101d70 0x7fb6ac102140 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 crypto rx=0x7fb6a0004940 tx=0x7fb6a0009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:05.671 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.640+0000 7fb6b231e700 1 -- 172.21.15.152:0/2942839204 shutdown_connections 2024-04-02T04:55:05.671 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.640+0000 7fb6b231e700 1 --2- 172.21.15.152:0/2942839204 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb6ac102710 0x7fb6ac0fbd80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:05.671 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.640+0000 7fb6b231e700 1 --2- 172.21.15.152:0/2942839204 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb6ac101d70 0x7fb6ac102140 unknown :-1 s=CLOSED pgs=199 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:05.671 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.640+0000 7fb6b231e700 1 -- 172.21.15.152:0/2942839204 >> 172.21.15.152:0/2942839204 conn(0x7fb6ac072bc0 msgr2=0x7fb6ac072fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:05.671 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.640+0000 7fb6b231e700 1 -- 172.21.15.152:0/2942839204 shutdown_connections 2024-04-02T04:55:05.671 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.640+0000 7fb6b231e700 1 -- 172.21.15.152:0/2942839204 wait complete. 2024-04-02T04:55:05.671 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.641+0000 7fb6b231e700 1 Processor -- start 2024-04-02T04:55:05.671 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.641+0000 7fb6b231e700 1 -- start start 2024-04-02T04:55:05.671 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.641+0000 7fb6b231e700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb6ac101d70 0x7fb6ac102bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:05.671 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.641+0000 7fb6b231e700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb6ac102710 0x7fb6ac103110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:05.671 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.641+0000 7fb6b231e700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb6ac1036a0 con 0x7fb6ac102710 2024-04-02T04:55:05.672 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.641+0000 7fb6b231e700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb6ac1037e0 con 0x7fb6ac101d70 2024-04-02T04:55:05.672 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.641+0000 7fb6ab7fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb6ac102710 0x7fb6ac103110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:05.672 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.641+0000 7fb6ab7fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb6ac102710 0x7fb6ac103110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:39334/0 (socket says 172.21.15.152:39334) 2024-04-02T04:55:05.672 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.641+0000 7fb6ab7fe700 1 -- 172.21.15.152:0/3949933789 learned_addr learned my addr 172.21.15.152:0/3949933789 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:55:05.672 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.641+0000 7fb6abfff700 1 --2- 172.21.15.152:0/3949933789 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb6ac101d70 0x7fb6ac102bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:05.672 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.641+0000 7fb6ab7fe700 1 -- 172.21.15.152:0/3949933789 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb6ac101d70 msgr2=0x7fb6ac102bd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:05.672 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.641+0000 7fb6ab7fe700 1 --2- 172.21.15.152:0/3949933789 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb6ac101d70 0x7fb6ac102bd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:05.672 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.641+0000 7fb6ab7fe700 1 -- 172.21.15.152:0/3949933789 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb6a0005040 con 0x7fb6ac102710 2024-04-02T04:55:05.672 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.641+0000 7fb6abfff700 1 --2- 172.21.15.152:0/3949933789 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb6ac101d70 0x7fb6ac102bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-02T04:55:05.672 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.642+0000 7fb6ab7fe700 1 --2- 172.21.15.152:0/3949933789 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb6ac102710 0x7fb6ac103110 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7fb69c00c800 tx=0x7fb69c004470 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:05.672 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.642+0000 7fb6a8ff9700 1 -- 172.21.15.152:0/3949933789 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fb69c00a760 con 0x7fb6ac102710 2024-04-02T04:55:05.672 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.642+0000 7fb6a8ff9700 1 -- 172.21.15.152:0/3949933789 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fb69c00ad90 con 0x7fb6ac102710 2024-04-02T04:55:05.672 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.642+0000 7fb6b231e700 1 -- 172.21.15.152:0/3949933789 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb6ac119e80 con 0x7fb6ac102710 2024-04-02T04:55:05.673 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.642+0000 7fb6a8ff9700 1 -- 172.21.15.152:0/3949933789 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fb69c0113a0 con 0x7fb6ac102710 2024-04-02T04:55:05.673 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.642+0000 7fb6b231e700 1 -- 172.21.15.152:0/3949933789 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb6ac11a300 con 0x7fb6ac102710 2024-04-02T04:55:05.673 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.643+0000 7fb6b231e700 1 -- 172.21.15.152:0/3949933789 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7fb6ac105fe0 con 0x7fb6ac102710 2024-04-02T04:55:05.673 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.643+0000 7fb6a8ff9700 1 -- 172.21.15.152:0/3949933789 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7fb69c011610 con 0x7fb6ac102710 2024-04-02T04:55:05.673 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.644+0000 7fb6a8ff9700 1 --2- 172.21.15.152:0/3949933789 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb6940692a0 0x7fb69406b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:05.673 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.644+0000 7fb6abfff700 1 --2- 172.21.15.152:0/3949933789 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb6940692a0 0x7fb69406b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:05.673 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.644+0000 7fb6a8ff9700 1 -- 172.21.15.152:0/3949933789 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7fb69c086eb0 con 0x7fb6ac102710 2024-04-02T04:55:05.674 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.644+0000 7fb6a8ff9700 1 --2- 172.21.15.152:0/3949933789 >> [v2:172.21.15.154:6808/220818867,v1:172.21.15.154:6809/220818867] conn(0x7fb69406ef70 0x7fb694071380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:05.674 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.644+0000 7fb6a8ff9700 1 -- 172.21.15.152:0/3949933789 --> [v2:172.21.15.154:6808/220818867,v1:172.21.15.154:6809/220818867] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7fb694071a30 con 0x7fb69406ef70 2024-04-02T04:55:05.674 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.644+0000 7fb6a8ff9700 1 -- 172.21.15.152:0/3949933789 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_get_version_reply(handle=1 version=40) v2 ==== 24+0+0 (secure 0 0 0) 0x7fb69c08c510 con 0x7fb6ac102710 2024-04-02T04:55:05.674 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.644+0000 7fb6abfff700 1 --2- 172.21.15.152:0/3949933789 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb6940692a0 0x7fb69406b750 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fb6a0004560 tx=0x7fb6a0009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:05.674 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.644+0000 7fb6b08bb700 1 --2- 172.21.15.152:0/3949933789 >> [v2:172.21.15.154:6808/220818867,v1:172.21.15.154:6809/220818867] conn(0x7fb69406ef70 0x7fb694071380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:05.674 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.645+0000 7fb6b08bb700 1 --2- 172.21.15.152:0/3949933789 >> [v2:172.21.15.154:6808/220818867,v1:172.21.15.154:6809/220818867] conn(0x7fb69406ef70 0x7fb694071380 crc :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.4 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:05.674 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.645+0000 7fb6a8ff9700 1 -- 172.21.15.152:0/3949933789 <== osd.4 v2:172.21.15.154:6808/220818867 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+24147 (crc 0 0 0) 0x7fb694071a30 con 0x7fb69406ef70 2024-04-02T04:55:05.674 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.660+0000 7fb6b231e700 1 -- 172.21.15.152:0/3949933789 --> [v2:172.21.15.154:6808/220818867,v1:172.21.15.154:6809/220818867] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7fb6ac0fd4c0 con 0x7fb69406ef70 2024-04-02T04:55:05.674 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.660+0000 7fb6a8ff9700 1 -- 172.21.15.152:0/3949933789 <== osd.4 v2:172.21.15.154:6808/220818867 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+12 (crc 0 0 0) 0x7fb6ac0fd4c0 con 0x7fb69406ef70 2024-04-02T04:55:05.674 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.661+0000 7fb6b231e700 1 -- 172.21.15.152:0/3949933789 >> [v2:172.21.15.154:6808/220818867,v1:172.21.15.154:6809/220818867] conn(0x7fb69406ef70 msgr2=0x7fb694071380 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:05.674 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.661+0000 7fb6b231e700 1 --2- 172.21.15.152:0/3949933789 >> [v2:172.21.15.154:6808/220818867,v1:172.21.15.154:6809/220818867] conn(0x7fb69406ef70 0x7fb694071380 crc :-1 s=READY pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:05.674 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.661+0000 7fb6b231e700 1 -- 172.21.15.152:0/3949933789 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb6940692a0 msgr2=0x7fb69406b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:05.674 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.661+0000 7fb6b231e700 1 --2- 172.21.15.152:0/3949933789 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb6940692a0 0x7fb69406b750 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 crypto rx=0x7fb6a0004560 tx=0x7fb6a0009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:05.675 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.661+0000 7fb6b231e700 1 -- 172.21.15.152:0/3949933789 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb6ac102710 msgr2=0x7fb6ac103110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:05.675 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.661+0000 7fb6b231e700 1 --2- 172.21.15.152:0/3949933789 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb6ac102710 0x7fb6ac103110 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 crypto rx=0x7fb69c00c800 tx=0x7fb69c004470 comp rx=0 tx=0).stop 2024-04-02T04:55:05.675 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.661+0000 7fb6b231e700 1 -- 172.21.15.152:0/3949933789 shutdown_connections 2024-04-02T04:55:05.675 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.661+0000 7fb6b231e700 1 --2- 172.21.15.152:0/3949933789 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb6940692a0 0x7fb69406b750 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:05.675 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.661+0000 7fb6b231e700 1 --2- 172.21.15.152:0/3949933789 >> [v2:172.21.15.154:6808/220818867,v1:172.21.15.154:6809/220818867] conn(0x7fb69406ef70 0x7fb694071380 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:05.675 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.661+0000 7fb6b231e700 1 --2- 172.21.15.152:0/3949933789 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb6ac101d70 0x7fb6ac102bd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:05.675 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.661+0000 7fb6b231e700 1 --2- 172.21.15.152:0/3949933789 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb6ac102710 0x7fb6ac103110 unknown :-1 s=CLOSED pgs=200 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:05.675 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.661+0000 7fb6b231e700 1 -- 172.21.15.152:0/3949933789 >> 172.21.15.152:0/3949933789 conn(0x7fb6ac072bc0 msgr2=0x7fb6ac065960 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:05.675 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.661+0000 7fb6b231e700 1 -- 172.21.15.152:0/3949933789 shutdown_connections 2024-04-02T04:55:05.675 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:05.661+0000 7fb6b231e700 1 -- 172.21.15.152:0/3949933789 wait complete. 2024-04-02T04:55:06.114 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.099+0000 7f88df920700 1 -- 172.21.15.152:0/3820972826 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f88d81037a0 msgr2=0x7f88d80fce10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:06.114 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.099+0000 7f88df920700 1 --2- 172.21.15.152:0/3820972826 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f88d81037a0 0x7f88d80fce10 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 crypto rx=0x7f88c8004940 tx=0x7f88c8009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:06.114 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.099+0000 7f88df920700 1 -- 172.21.15.152:0/3820972826 shutdown_connections 2024-04-02T04:55:06.115 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.099+0000 7f88df920700 1 --2- 172.21.15.152:0/3820972826 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f88d81037a0 0x7f88d80fce10 unknown :-1 s=CLOSED pgs=201 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:06.115 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.099+0000 7f88df920700 1 --2- 172.21.15.152:0/3820972826 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f88d8102e00 0x7f88d81031d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:06.115 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.099+0000 7f88df920700 1 -- 172.21.15.152:0/3820972826 >> 172.21.15.152:0/3820972826 conn(0x7f88d80f89b0 msgr2=0x7f88d80fadc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:06.115 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.099+0000 7f88df920700 1 -- 172.21.15.152:0/3820972826 shutdown_connections 2024-04-02T04:55:06.115 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.099+0000 7f88df920700 1 -- 172.21.15.152:0/3820972826 wait complete. 2024-04-02T04:55:06.115 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.099+0000 7f88df920700 1 Processor -- start 2024-04-02T04:55:06.115 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.099+0000 7f88df920700 1 -- start start 2024-04-02T04:55:06.115 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.100+0000 7f88df920700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f88d8102e00 0x7f88d8113980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:06.115 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.100+0000 7f88df920700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f88d81037a0 0x7f88d8113ec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:06.115 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.100+0000 7f88df920700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f88d8114510 con 0x7f88d81037a0 2024-04-02T04:55:06.115 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.100+0000 7f88df920700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f88d810db10 con 0x7f88d8102e00 2024-04-02T04:55:06.115 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.100+0000 7f88dcebb700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f88d81037a0 0x7f88d8113ec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:06.115 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.100+0000 7f88dcebb700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f88d81037a0 0x7f88d8113ec0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:39356/0 (socket says 172.21.15.152:39356) 2024-04-02T04:55:06.115 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.100+0000 7f88dcebb700 1 -- 172.21.15.152:0/2048950236 learned_addr learned my addr 172.21.15.152:0/2048950236 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:55:06.116 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.100+0000 7f88dcebb700 1 -- 172.21.15.152:0/2048950236 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f88d8102e00 msgr2=0x7f88d8113980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:06.116 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.100+0000 7f88dd6bc700 1 --2- 172.21.15.152:0/2048950236 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f88d8102e00 0x7f88d8113980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:06.116 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.100+0000 7f88dcebb700 1 --2- 172.21.15.152:0/2048950236 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f88d8102e00 0x7f88d8113980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:06.116 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.100+0000 7f88dcebb700 1 -- 172.21.15.152:0/2048950236 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f88c8005040 con 0x7f88d81037a0 2024-04-02T04:55:06.116 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.100+0000 7f88dd6bc700 1 --2- 172.21.15.152:0/2048950236 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f88d8102e00 0x7f88d8113980 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-02T04:55:06.116 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.100+0000 7f88dcebb700 1 --2- 172.21.15.152:0/2048950236 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f88d81037a0 0x7f88d8113ec0 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f88c8009100 tx=0x7f88c8009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:06.116 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.100+0000 7f88cdffb700 1 -- 172.21.15.152:0/2048950236 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f88c800e040 con 0x7f88d81037a0 2024-04-02T04:55:06.116 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.100+0000 7f88cdffb700 1 -- 172.21.15.152:0/2048950236 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f88c8013420 con 0x7f88d81037a0 2024-04-02T04:55:06.116 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.100+0000 7f88df920700 1 -- 172.21.15.152:0/2048950236 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f88d810dd90 con 0x7f88d81037a0 2024-04-02T04:55:06.116 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.100+0000 7f88cdffb700 1 -- 172.21.15.152:0/2048950236 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f88c801c610 con 0x7f88d81037a0 2024-04-02T04:55:06.116 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.101+0000 7f88df920700 1 -- 172.21.15.152:0/2048950236 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f88d810e350 con 0x7f88d81037a0 2024-04-02T04:55:06.116 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.101+0000 7f88df920700 1 -- 172.21.15.152:0/2048950236 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_get_version(what=osdmap handle=1) v1 -- 0x7f88d810e830 con 0x7f88d81037a0 2024-04-02T04:55:06.116 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.102+0000 7f88cdffb700 1 -- 172.21.15.152:0/2048950236 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f88c8013590 con 0x7f88d81037a0 2024-04-02T04:55:06.116 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.102+0000 7f88cdffb700 1 --2- 172.21.15.152:0/2048950236 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f88c40692a0 0x7f88c406b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:06.117 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.102+0000 7f88cdffb700 1 -- 172.21.15.152:0/2048950236 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7f88c8088b30 con 0x7f88d81037a0 2024-04-02T04:55:06.117 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.102+0000 7f88dd6bc700 1 --2- 172.21.15.152:0/2048950236 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f88c40692a0 0x7f88c406b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:06.117 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.102+0000 7f88cdffb700 1 --2- 172.21.15.152:0/2048950236 >> [v2:172.21.15.152:6810/782616861,v1:172.21.15.152:6811/782616861] conn(0x7f88c406ef70 0x7f88c4071380 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:06.117 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.102+0000 7f88cdffb700 1 -- 172.21.15.152:0/2048950236 --> [v2:172.21.15.152:6810/782616861,v1:172.21.15.152:6811/782616861] -- command(tid 1: {"prefix": "get_command_descriptions"}) v1 -- 0x7f88c4071a30 con 0x7f88c406ef70 2024-04-02T04:55:06.117 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.102+0000 7f88cdffb700 1 -- 172.21.15.152:0/2048950236 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_get_version_reply(handle=1 version=40) v2 ==== 24+0+0 (secure 0 0 0) 0x7f88c8088f10 con 0x7f88d81037a0 2024-04-02T04:55:06.117 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.103+0000 7f88ddebd700 1 --2- 172.21.15.152:0/2048950236 >> [v2:172.21.15.152:6810/782616861,v1:172.21.15.152:6811/782616861] conn(0x7f88c406ef70 0x7f88c4071380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:06.117 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.103+0000 7f88dd6bc700 1 --2- 172.21.15.152:0/2048950236 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f88c40692a0 0x7f88c406b750 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f88d40041d0 tx=0x7f88d4009040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:06.117 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.103+0000 7f88ddebd700 1 --2- 172.21.15.152:0/2048950236 >> [v2:172.21.15.152:6810/782616861,v1:172.21.15.152:6811/782616861] conn(0x7f88c406ef70 0x7f88c4071380 crc :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).ready entity=osd.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:06.117 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.104+0000 7f88cdffb700 1 -- 172.21.15.152:0/2048950236 <== osd.1 v2:172.21.15.152:6810/782616861 1 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+24147 (crc 0 0 0) 0x7f88c4071a30 con 0x7f88c406ef70 2024-04-02T04:55:06.123 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.118+0000 7f88df920700 1 -- 172.21.15.152:0/2048950236 --> [v2:172.21.15.152:6810/782616861,v1:172.21.15.152:6811/782616861] -- command(tid 2: {"prefix": "flush_pg_stats"}) v1 -- 0x7f88d805e6c0 con 0x7f88c406ef70 2024-04-02T04:55:06.123 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.119+0000 7f88cdffb700 1 -- 172.21.15.152:0/2048950236 <== osd.1 v2:172.21.15.152:6810/782616861 2 ==== command_reply(tid 2: 0 ) v1 ==== 8+0+11 (crc 0 0 0) 0x7f88d805e6c0 con 0x7f88c406ef70 2024-04-02T04:55:06.124 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.119+0000 7f88df920700 1 -- 172.21.15.152:0/2048950236 >> [v2:172.21.15.152:6810/782616861,v1:172.21.15.152:6811/782616861] conn(0x7f88c406ef70 msgr2=0x7f88c4071380 crc :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:06.124 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.119+0000 7f88df920700 1 --2- 172.21.15.152:0/2048950236 >> [v2:172.21.15.152:6810/782616861,v1:172.21.15.152:6811/782616861] conn(0x7f88c406ef70 0x7f88c4071380 crc :-1 s=READY pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:06.124 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.119+0000 7f88df920700 1 -- 172.21.15.152:0/2048950236 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f88c40692a0 msgr2=0x7f88c406b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:06.124 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.119+0000 7f88df920700 1 --2- 172.21.15.152:0/2048950236 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f88c40692a0 0x7f88c406b750 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 crypto rx=0x7f88d40041d0 tx=0x7f88d4009040 comp rx=0 tx=0).stop 2024-04-02T04:55:06.125 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.119+0000 7f88df920700 1 -- 172.21.15.152:0/2048950236 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f88d81037a0 msgr2=0x7f88d8113ec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:06.125 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.119+0000 7f88df920700 1 --2- 172.21.15.152:0/2048950236 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f88d81037a0 0x7f88d8113ec0 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 crypto rx=0x7f88c8009100 tx=0x7f88c8009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:06.125 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.119+0000 7f88df920700 1 -- 172.21.15.152:0/2048950236 shutdown_connections 2024-04-02T04:55:06.125 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.119+0000 7f88df920700 1 --2- 172.21.15.152:0/2048950236 >> [v2:172.21.15.152:6810/782616861,v1:172.21.15.152:6811/782616861] conn(0x7f88c406ef70 0x7f88c4071380 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:06.125 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.119+0000 7f88df920700 1 --2- 172.21.15.152:0/2048950236 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f88c40692a0 0x7f88c406b750 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:06.125 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.119+0000 7f88df920700 1 --2- 172.21.15.152:0/2048950236 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f88d8102e00 0x7f88d8113980 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:06.125 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.119+0000 7f88df920700 1 --2- 172.21.15.152:0/2048950236 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f88d81037a0 0x7f88d8113ec0 unknown :-1 s=CLOSED pgs=202 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:06.125 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.119+0000 7f88df920700 1 -- 172.21.15.152:0/2048950236 >> 172.21.15.152:0/2048950236 conn(0x7f88d80f89b0 msgr2=0x7f88d81021d0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:06.125 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.119+0000 7f88df920700 1 -- 172.21.15.152:0/2048950236 shutdown_connections 2024-04-02T04:55:06.125 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:06.119+0000 7f88df920700 1 -- 172.21.15.152:0/2048950236 wait complete. 2024-04-02T04:55:06.627 INFO:teuthology.orchestra.run.smithi152.stdout:146028888079 2024-04-02T04:55:06.628 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.4 2024-04-02T04:55:07.065 INFO:teuthology.orchestra.run.smithi152.stdout:167503724553 2024-04-02T04:55:07.065 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.5 2024-04-02T04:55:07.351 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:07 smithi152 ceph-mon[29073]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:07.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:07 smithi154 ceph-mon[35729]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:07.741 INFO:teuthology.orchestra.run.smithi152.stdout:60129542173 2024-04-02T04:55:07.742 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.1 2024-04-02T04:55:08.451 INFO:teuthology.orchestra.run.smithi152.stdout:38654705700 2024-04-02T04:55:08.451 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph osd last-stat-seq osd.0 2024-04-02T04:55:08.510 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:55:08.510 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:55:08.558 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:55:09.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:09 smithi152 ceph-mon[29073]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:09.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:09 smithi154 ceph-mon[35729]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:10.218 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:10 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:55:10.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:10 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:55:11.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:11 smithi152 ceph-mon[29073]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:11.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:11 smithi154 ceph-mon[35729]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:12.250 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.223+0000 7fd2b469c700 1 -- 172.21.15.152:0/213498071 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd2ac1037c0 msgr2=0x7fd2ac0fce30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:12.250 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.223+0000 7fd2b469c700 1 --2- 172.21.15.152:0/213498071 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd2ac1037c0 0x7fd2ac0fce30 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 crypto rx=0x7fd2a0004940 tx=0x7fd2a0009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:12.250 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.223+0000 7fd2b469c700 1 -- 172.21.15.152:0/213498071 shutdown_connections 2024-04-02T04:55:12.250 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.223+0000 7fd2b469c700 1 --2- 172.21.15.152:0/213498071 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd2ac1037c0 0x7fd2ac0fce30 unknown :-1 s=CLOSED pgs=205 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:12.250 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.223+0000 7fd2b469c700 1 --2- 172.21.15.152:0/213498071 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd2ac102e20 0x7fd2ac1031f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:12.250 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.223+0000 7fd2b469c700 1 -- 172.21.15.152:0/213498071 >> 172.21.15.152:0/213498071 conn(0x7fd2ac0f89d0 msgr2=0x7fd2ac0fade0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:12.250 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.223+0000 7fd2b469c700 1 -- 172.21.15.152:0/213498071 shutdown_connections 2024-04-02T04:55:12.251 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.223+0000 7fd2b469c700 1 -- 172.21.15.152:0/213498071 wait complete. 2024-04-02T04:55:12.251 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.224+0000 7fd2b469c700 1 Processor -- start 2024-04-02T04:55:12.251 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.224+0000 7fd2b469c700 1 -- start start 2024-04-02T04:55:12.251 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.224+0000 7fd2b469c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd2ac102e20 0x7fd2ac113b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:12.251 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.224+0000 7fd2b469c700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd2ac1037c0 0x7fd2ac1140c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:12.251 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.224+0000 7fd2b469c700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd2ac1147a0 con 0x7fd2ac102e20 2024-04-02T04:55:12.251 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.224+0000 7fd2b469c700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd2ac10db20 con 0x7fd2ac1037c0 2024-04-02T04:55:12.251 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.224+0000 7fd2b2438700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd2ac102e20 0x7fd2ac113b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:12.251 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.224+0000 7fd2b2438700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd2ac102e20 0x7fd2ac113b80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:45772/0 (socket says 172.21.15.152:45772) 2024-04-02T04:55:12.251 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.224+0000 7fd2b2438700 1 -- 172.21.15.152:0/461198823 learned_addr learned my addr 172.21.15.152:0/461198823 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:55:12.251 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.224+0000 7fd2b1c37700 1 --2- 172.21.15.152:0/461198823 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd2ac1037c0 0x7fd2ac1140c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:12.251 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.224+0000 7fd2b2438700 1 -- 172.21.15.152:0/461198823 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd2ac1037c0 msgr2=0x7fd2ac1140c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:12.251 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.224+0000 7fd2b2438700 1 --2- 172.21.15.152:0/461198823 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd2ac1037c0 0x7fd2ac1140c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:12.251 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.224+0000 7fd2b2438700 1 -- 172.21.15.152:0/461198823 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd2a0005040 con 0x7fd2ac102e20 2024-04-02T04:55:12.252 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.224+0000 7fd2b2438700 1 --2- 172.21.15.152:0/461198823 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd2ac102e20 0x7fd2ac113b80 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7fd2a400b790 tx=0x7fd2a4009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:12.252 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.225+0000 7fd2aaffd700 1 -- 172.21.15.152:0/461198823 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fd2a400f770 con 0x7fd2ac102e20 2024-04-02T04:55:12.252 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.225+0000 7fd2aaffd700 1 -- 172.21.15.152:0/461198823 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fd2a400fda0 con 0x7fd2ac102e20 2024-04-02T04:55:12.252 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.225+0000 7fd2b469c700 1 -- 172.21.15.152:0/461198823 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd2ac10de00 con 0x7fd2ac102e20 2024-04-02T04:55:12.252 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.225+0000 7fd2aaffd700 1 -- 172.21.15.152:0/461198823 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fd2a400d310 con 0x7fd2ac102e20 2024-04-02T04:55:12.252 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.225+0000 7fd2b469c700 1 -- 172.21.15.152:0/461198823 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd2ac10e3c0 con 0x7fd2ac102e20 2024-04-02T04:55:12.252 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.226+0000 7fd2b469c700 1 -- 172.21.15.152:0/461198823 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd2ac0fe570 con 0x7fd2ac102e20 2024-04-02T04:55:12.252 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.226+0000 7fd2aaffd700 1 -- 172.21.15.152:0/461198823 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7fd2a4010420 con 0x7fd2ac102e20 2024-04-02T04:55:12.252 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.227+0000 7fd2aaffd700 1 --2- 172.21.15.152:0/461198823 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd2940692a0 0x7fd29406b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:12.252 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.227+0000 7fd2b1c37700 1 --2- 172.21.15.152:0/461198823 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd2940692a0 0x7fd29406b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:12.252 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.227+0000 7fd2aaffd700 1 -- 172.21.15.152:0/461198823 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7fd2a4086d60 con 0x7fd2ac102e20 2024-04-02T04:55:12.252 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.227+0000 7fd2b1c37700 1 --2- 172.21.15.152:0/461198823 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd2940692a0 0x7fd29406b750 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7fd2a000b3b0 tx=0x7fd2a0009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:12.252 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.231+0000 7fd2aaffd700 1 -- 172.21.15.152:0/461198823 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fd2a4053470 con 0x7fd2ac102e20 2024-04-02T04:55:12.367 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.137+0000 7f685150f700 1 -- 172.21.15.152:0/673609317 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f684c0fbc80 msgr2=0x7f684c0fc050 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:12.367 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.137+0000 7f685150f700 1 --2- 172.21.15.152:0/673609317 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f684c0fbc80 0x7f684c0fc050 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 crypto rx=0x7f683c004940 tx=0x7f683c009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:12.367 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.137+0000 7f685150f700 1 -- 172.21.15.152:0/673609317 shutdown_connections 2024-04-02T04:55:12.367 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.137+0000 7f685150f700 1 --2- 172.21.15.152:0/673609317 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f684c0fc590 0x7f684c1092f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:12.367 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.137+0000 7f685150f700 1 --2- 172.21.15.152:0/673609317 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f684c0fbc80 0x7f684c0fc050 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:12.367 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.137+0000 7f685150f700 1 -- 172.21.15.152:0/673609317 >> 172.21.15.152:0/673609317 conn(0x7f684c072bc0 msgr2=0x7f684c072fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:12.367 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.137+0000 7f685150f700 1 -- 172.21.15.152:0/673609317 shutdown_connections 2024-04-02T04:55:12.368 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.137+0000 7f685150f700 1 -- 172.21.15.152:0/673609317 wait complete. 2024-04-02T04:55:12.368 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.138+0000 7f685150f700 1 Processor -- start 2024-04-02T04:55:12.368 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.138+0000 7f685150f700 1 -- start start 2024-04-02T04:55:12.368 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.138+0000 7f685150f700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f684c0fbc80 0x7f684c115ca0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:12.368 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.138+0000 7f685150f700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f684c0fc590 0x7f684c1161e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:12.368 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.138+0000 7f685150f700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f684c116830 con 0x7f684c0fc590 2024-04-02T04:55:12.368 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.138+0000 7f685150f700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f684c10fc50 con 0x7f684c0fbc80 2024-04-02T04:55:12.368 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.138+0000 7f684a7fc700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f684c0fc590 0x7f684c1161e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:12.368 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.138+0000 7f684a7fc700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f684c0fc590 0x7f684c1161e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:45744/0 (socket says 172.21.15.152:45744) 2024-04-02T04:55:12.368 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.138+0000 7f684a7fc700 1 -- 172.21.15.152:0/964881702 learned_addr learned my addr 172.21.15.152:0/964881702 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:55:12.368 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.139+0000 7f684affd700 1 --2- 172.21.15.152:0/964881702 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f684c0fbc80 0x7f684c115ca0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:12.368 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.139+0000 7f684a7fc700 1 -- 172.21.15.152:0/964881702 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f684c0fbc80 msgr2=0x7f684c115ca0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:12.368 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.139+0000 7f684a7fc700 1 --2- 172.21.15.152:0/964881702 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f684c0fbc80 0x7f684c115ca0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:12.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.139+0000 7f684a7fc700 1 -- 172.21.15.152:0/964881702 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f683c005040 con 0x7f684c0fc590 2024-04-02T04:55:12.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.139+0000 7f684a7fc700 1 --2- 172.21.15.152:0/964881702 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f684c0fc590 0x7f684c1161e0 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f684000eb40 tx=0x7f684000b040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:12.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.139+0000 7f68337fe700 1 -- 172.21.15.152:0/964881702 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f6840007cc0 con 0x7f684c0fc590 2024-04-02T04:55:12.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.139+0000 7f68337fe700 1 -- 172.21.15.152:0/964881702 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f684000f040 con 0x7f684c0fc590 2024-04-02T04:55:12.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.139+0000 7f685150f700 1 -- 172.21.15.152:0/964881702 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f684c10fed0 con 0x7f684c0fc590 2024-04-02T04:55:12.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.139+0000 7f68337fe700 1 -- 172.21.15.152:0/964881702 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f684000ca10 con 0x7f684c0fc590 2024-04-02T04:55:12.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.139+0000 7f685150f700 1 -- 172.21.15.152:0/964881702 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f684c110410 con 0x7f684c0fc590 2024-04-02T04:55:12.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.140+0000 7f685150f700 1 -- 172.21.15.152:0/964881702 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f684c065e30 con 0x7f684c0fc590 2024-04-02T04:55:12.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.141+0000 7f68337fe700 1 -- 172.21.15.152:0/964881702 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f684000cc80 con 0x7f684c0fc590 2024-04-02T04:55:12.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.141+0000 7f68337fe700 1 --2- 172.21.15.152:0/964881702 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f68340692a0 0x7f683406b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:12.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.141+0000 7f68337fe700 1 -- 172.21.15.152:0/964881702 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7f6840086a60 con 0x7f684c0fc590 2024-04-02T04:55:12.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.141+0000 7f684affd700 1 --2- 172.21.15.152:0/964881702 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f68340692a0 0x7f683406b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:12.370 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.142+0000 7f684affd700 1 --2- 172.21.15.152:0/964881702 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f68340692a0 0x7f683406b750 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f683c009b30 tx=0x7f683c009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:12.370 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.146+0000 7f68337fe700 1 -- 172.21.15.152:0/964881702 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f684004e190 con 0x7f684c0fc590 2024-04-02T04:55:12.370 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.339+0000 7f685150f700 1 -- 172.21.15.152:0/964881702 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 3} v 0) v1 -- 0x7f684c10c160 con 0x7f684c0fc590 2024-04-02T04:55:12.370 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.339+0000 7f68337fe700 1 -- 172.21.15.152:0/964881702 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 3}]=0 v0) v1 ==== 74+0+13 (secure 0 0 0) 0x7f6840056b90 con 0x7f684c0fc590 2024-04-02T04:55:12.371 INFO:teuthology.orchestra.run.smithi152.stdout:111669149716 2024-04-02T04:55:12.371 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.341+0000 7f685150f700 1 -- 172.21.15.152:0/964881702 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f68340692a0 msgr2=0x7f683406b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:12.371 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.341+0000 7f685150f700 1 --2- 172.21.15.152:0/964881702 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f68340692a0 0x7f683406b750 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 crypto rx=0x7f683c009b30 tx=0x7f683c009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:12.371 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.341+0000 7f685150f700 1 -- 172.21.15.152:0/964881702 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f684c0fc590 msgr2=0x7f684c1161e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:12.371 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.341+0000 7f685150f700 1 --2- 172.21.15.152:0/964881702 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f684c0fc590 0x7f684c1161e0 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 crypto rx=0x7f684000eb40 tx=0x7f684000b040 comp rx=0 tx=0).stop 2024-04-02T04:55:12.371 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.341+0000 7f685150f700 1 -- 172.21.15.152:0/964881702 shutdown_connections 2024-04-02T04:55:12.371 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.341+0000 7f685150f700 1 --2- 172.21.15.152:0/964881702 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f68340692a0 0x7f683406b750 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:12.371 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.341+0000 7f685150f700 1 --2- 172.21.15.152:0/964881702 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f684c0fbc80 0x7f684c115ca0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:12.371 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.341+0000 7f685150f700 1 --2- 172.21.15.152:0/964881702 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f684c0fc590 0x7f684c1161e0 unknown :-1 s=CLOSED pgs=204 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:12.371 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.341+0000 7f685150f700 1 -- 172.21.15.152:0/964881702 >> 172.21.15.152:0/964881702 conn(0x7f684c072bc0 msgr2=0x7f684c105ad0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:12.371 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.341+0000 7f685150f700 1 -- 172.21.15.152:0/964881702 shutdown_connections 2024-04-02T04:55:12.372 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.341+0000 7f685150f700 1 -- 172.21.15.152:0/964881702 wait complete. 2024-04-02T04:55:12.435 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.431+0000 7fd2b469c700 1 -- 172.21.15.152:0/461198823 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 5} v 0) v1 -- 0x7fd2ac04bed0 con 0x7fd2ac102e20 2024-04-02T04:55:12.435 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.431+0000 7fd2aaffd700 1 -- 172.21.15.152:0/461198823 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 5}]=0 v0) v1 ==== 74+0+13 (secure 0 0 0) 0x7fd2a4056a90 con 0x7fd2ac102e20 2024-04-02T04:55:12.435 INFO:teuthology.orchestra.run.smithi152.stdout:167503724554 2024-04-02T04:55:12.437 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.433+0000 7fd2b469c700 1 -- 172.21.15.152:0/461198823 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd2940692a0 msgr2=0x7fd29406b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:12.437 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.433+0000 7fd2b469c700 1 --2- 172.21.15.152:0/461198823 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd2940692a0 0x7fd29406b750 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 crypto rx=0x7fd2a000b3b0 tx=0x7fd2a0009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:12.437 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.433+0000 7fd2b469c700 1 -- 172.21.15.152:0/461198823 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd2ac102e20 msgr2=0x7fd2ac113b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:12.437 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.433+0000 7fd2b469c700 1 --2- 172.21.15.152:0/461198823 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd2ac102e20 0x7fd2ac113b80 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 crypto rx=0x7fd2a400b790 tx=0x7fd2a4009250 comp rx=0 tx=0).stop 2024-04-02T04:55:12.437 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.433+0000 7fd2b469c700 1 -- 172.21.15.152:0/461198823 shutdown_connections 2024-04-02T04:55:12.437 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.433+0000 7fd2b469c700 1 --2- 172.21.15.152:0/461198823 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd2940692a0 0x7fd29406b750 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:12.437 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.433+0000 7fd2b469c700 1 --2- 172.21.15.152:0/461198823 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd2ac102e20 0x7fd2ac113b80 unknown :-1 s=CLOSED pgs=206 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:12.437 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.433+0000 7fd2b469c700 1 --2- 172.21.15.152:0/461198823 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd2ac1037c0 0x7fd2ac1140c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:12.437 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.433+0000 7fd2b469c700 1 -- 172.21.15.152:0/461198823 >> 172.21.15.152:0/461198823 conn(0x7fd2ac0f89d0 msgr2=0x7fd2ac0f99a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:12.437 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.433+0000 7fd2b469c700 1 -- 172.21.15.152:0/461198823 shutdown_connections 2024-04-02T04:55:12.437 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:12.433+0000 7fd2b469c700 1 -- 172.21.15.152:0/461198823 wait complete. 2024-04-02T04:55:12.608 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:55:12.608 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:55:13.258 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:13 smithi152 ceph-mon[29073]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:13.258 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:13 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/964881702' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-02T04:55:13.258 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:13 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/461198823' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-02T04:55:13.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:13 smithi154 ceph-mon[35729]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:13.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:13 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/964881702' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-02T04:55:13.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:13 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/461198823' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-02T04:55:13.686 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724553 got 167503724554 for osd.5 2024-04-02T04:55:13.686 DEBUG:teuthology.parallel:result is None 2024-04-02T04:55:14.098 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.093+0000 7f0e51d42700 1 -- 172.21.15.152:0/1202955501 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0e4c102710 msgr2=0x7f0e4c0fbd80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:14.098 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.093+0000 7f0e51d42700 1 --2- 172.21.15.152:0/1202955501 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0e4c102710 0x7f0e4c0fbd80 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 crypto rx=0x7f0e34004940 tx=0x7f0e34009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:14.099 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.093+0000 7f0e51d42700 1 -- 172.21.15.152:0/1202955501 shutdown_connections 2024-04-02T04:55:14.099 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.093+0000 7f0e51d42700 1 --2- 172.21.15.152:0/1202955501 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0e4c102710 0x7f0e4c0fbd80 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:14.099 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.093+0000 7f0e51d42700 1 --2- 172.21.15.152:0/1202955501 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0e4c101d70 0x7f0e4c102140 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:14.099 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.093+0000 7f0e51d42700 1 -- 172.21.15.152:0/1202955501 >> 172.21.15.152:0/1202955501 conn(0x7f0e4c072bc0 msgr2=0x7f0e4c072fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:14.099 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.093+0000 7f0e51d42700 1 -- 172.21.15.152:0/1202955501 shutdown_connections 2024-04-02T04:55:14.099 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.093+0000 7f0e51d42700 1 -- 172.21.15.152:0/1202955501 wait complete. 2024-04-02T04:55:14.099 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.093+0000 7f0e51d42700 1 Processor -- start 2024-04-02T04:55:14.099 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.094+0000 7f0e51d42700 1 -- start start 2024-04-02T04:55:14.099 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.094+0000 7f0e51d42700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0e4c101d70 0x7f0e4c113c00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:14.099 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.094+0000 7f0e51d42700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0e4c102710 0x7f0e4c114140 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:14.099 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.094+0000 7f0e51d42700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0e4c114790 con 0x7f0e4c101d70 2024-04-02T04:55:14.099 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.094+0000 7f0e51d42700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0e4c10db80 con 0x7f0e4c102710 2024-04-02T04:55:14.099 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.094+0000 7f0e4b7fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0e4c101d70 0x7f0e4c113c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:14.100 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.094+0000 7f0e4b7fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0e4c101d70 0x7f0e4c113c00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:45800/0 (socket says 172.21.15.152:45800) 2024-04-02T04:55:14.100 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.094+0000 7f0e4b7fe700 1 -- 172.21.15.152:0/2879599713 learned_addr learned my addr 172.21.15.152:0/2879599713 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:55:14.100 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.094+0000 7f0e42ffd700 1 --2- 172.21.15.152:0/2879599713 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0e4c102710 0x7f0e4c114140 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:14.100 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.094+0000 7f0e4b7fe700 1 -- 172.21.15.152:0/2879599713 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0e4c102710 msgr2=0x7f0e4c114140 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:14.100 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.094+0000 7f0e4b7fe700 1 --2- 172.21.15.152:0/2879599713 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0e4c102710 0x7f0e4c114140 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:14.100 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.094+0000 7f0e4b7fe700 1 -- 172.21.15.152:0/2879599713 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0e34005040 con 0x7f0e4c101d70 2024-04-02T04:55:14.100 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.094+0000 7f0e4b7fe700 1 --2- 172.21.15.152:0/2879599713 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0e4c101d70 0x7f0e4c113c00 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f0e3c002a50 tx=0x7f0e3c0046b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:14.100 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.095+0000 7f0e48ff9700 1 -- 172.21.15.152:0/2879599713 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f0e3c007cc0 con 0x7f0e4c101d70 2024-04-02T04:55:14.100 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.095+0000 7f0e51d42700 1 -- 172.21.15.152:0/2879599713 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0e4c10de60 con 0x7f0e4c101d70 2024-04-02T04:55:14.100 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.095+0000 7f0e48ff9700 1 -- 172.21.15.152:0/2879599713 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f0e3c00f040 con 0x7f0e4c101d70 2024-04-02T04:55:14.100 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.095+0000 7f0e51d42700 1 -- 172.21.15.152:0/2879599713 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0e4c10e3f0 con 0x7f0e4c101d70 2024-04-02T04:55:14.100 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.095+0000 7f0e48ff9700 1 -- 172.21.15.152:0/2879599713 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f0e3c00da70 con 0x7f0e4c101d70 2024-04-02T04:55:14.101 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.096+0000 7f0e51d42700 1 -- 172.21.15.152:0/2879599713 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0e2c0052b0 con 0x7f0e4c101d70 2024-04-02T04:55:14.101 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.096+0000 7f0e48ff9700 1 -- 172.21.15.152:0/2879599713 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f0e3c00dcb0 con 0x7f0e4c101d70 2024-04-02T04:55:14.101 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.096+0000 7f0e48ff9700 1 --2- 172.21.15.152:0/2879599713 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f0e380692a0 0x7f0e3806b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:14.101 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.097+0000 7f0e48ff9700 1 -- 172.21.15.152:0/2879599713 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7f0e3c0863e0 con 0x7f0e4c101d70 2024-04-02T04:55:14.101 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.097+0000 7f0e42ffd700 1 --2- 172.21.15.152:0/2879599713 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f0e380692a0 0x7f0e3806b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:14.101 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.097+0000 7f0e42ffd700 1 --2- 172.21.15.152:0/2879599713 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f0e380692a0 0x7f0e3806b750 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f0e4c10ee60 tx=0x7f0e34009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:14.104 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.101+0000 7f0e48ff9700 1 -- 172.21.15.152:0/2879599713 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f0e3c0564e0 con 0x7f0e4c101d70 2024-04-02T04:55:14.297 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.293+0000 7f0e51d42700 1 -- 172.21.15.152:0/2879599713 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 4} v 0) v1 -- 0x7f0e2c005560 con 0x7f0e4c101d70 2024-04-02T04:55:14.297 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.294+0000 7f0e48ff9700 1 -- 172.21.15.152:0/2879599713 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 4}]=0 v0) v1 ==== 74+0+13 (secure 0 0 0) 0x7f0e3c056070 con 0x7f0e4c101d70 2024-04-02T04:55:14.297 INFO:teuthology.orchestra.run.smithi152.stdout:146028888081 2024-04-02T04:55:14.299 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.295+0000 7f0e51d42700 1 -- 172.21.15.152:0/2879599713 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f0e380692a0 msgr2=0x7f0e3806b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:14.299 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.295+0000 7f0e51d42700 1 --2- 172.21.15.152:0/2879599713 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f0e380692a0 0x7f0e3806b750 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 crypto rx=0x7f0e4c10ee60 tx=0x7f0e34009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:14.299 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.296+0000 7f0e51d42700 1 -- 172.21.15.152:0/2879599713 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0e4c101d70 msgr2=0x7f0e4c113c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:14.299 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.296+0000 7f0e51d42700 1 --2- 172.21.15.152:0/2879599713 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0e4c101d70 0x7f0e4c113c00 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 crypto rx=0x7f0e3c002a50 tx=0x7f0e3c0046b0 comp rx=0 tx=0).stop 2024-04-02T04:55:14.299 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.296+0000 7f0e51d42700 1 -- 172.21.15.152:0/2879599713 shutdown_connections 2024-04-02T04:55:14.300 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.296+0000 7f0e51d42700 1 --2- 172.21.15.152:0/2879599713 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f0e380692a0 0x7f0e3806b750 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:14.300 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.296+0000 7f0e51d42700 1 --2- 172.21.15.152:0/2879599713 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0e4c101d70 0x7f0e4c113c00 unknown :-1 s=CLOSED pgs=208 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:14.300 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.296+0000 7f0e51d42700 1 --2- 172.21.15.152:0/2879599713 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0e4c102710 0x7f0e4c114140 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:14.300 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.296+0000 7f0e51d42700 1 -- 172.21.15.152:0/2879599713 >> 172.21.15.152:0/2879599713 conn(0x7f0e4c072bc0 msgr2=0x7f0e4c0659a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:14.300 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.296+0000 7f0e51d42700 1 -- 172.21.15.152:0/2879599713 shutdown_connections 2024-04-02T04:55:14.300 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:14.296+0000 7f0e51d42700 1 -- 172.21.15.152:0/2879599713 wait complete. 2024-04-02T04:55:15.005 INFO:tasks.cephadm.ceph_manager.ceph:need seq 146028888079 got 146028888081 for osd.4 2024-04-02T04:55:15.005 DEBUG:teuthology.parallel:result is None 2024-04-02T04:55:15.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:15 smithi154 ceph-mon[35729]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:15.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:15 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/2879599713' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-02T04:55:15.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:15 smithi152 ceph-mon[29073]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:15.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:15 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/2879599713' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-02T04:55:16.571 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:55:16.599 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149714 got 111669149716 for osd.3 2024-04-02T04:55:16.599 DEBUG:teuthology.parallel:result is None 2024-04-02T04:55:17.256 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:17 smithi152 ceph-mon[29073]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:17.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:17 smithi154 ceph-mon[35729]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:19.238 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.231+0000 7fcbe3ed4700 1 -- 172.21.15.152:0/249389745 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcbdc0fb3b0 msgr2=0x7fcbdc0fb820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:19.238 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.231+0000 7fcbe3ed4700 1 --2- 172.21.15.152:0/249389745 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcbdc0fb3b0 0x7fcbdc0fb820 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 crypto rx=0x7fcbcc004940 tx=0x7fcbcc009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:19.239 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.231+0000 7fcbe3ed4700 1 -- 172.21.15.152:0/249389745 shutdown_connections 2024-04-02T04:55:19.239 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.231+0000 7fcbe3ed4700 1 --2- 172.21.15.152:0/249389745 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcbdc0fb3b0 0x7fcbdc0fb820 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:19.239 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.231+0000 7fcbe3ed4700 1 --2- 172.21.15.152:0/249389745 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcbdc102f70 0x7fcbdc103340 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:19.239 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.231+0000 7fcbe3ed4700 1 -- 172.21.15.152:0/249389745 >> 172.21.15.152:0/249389745 conn(0x7fcbdc072310 msgr2=0x7fcbdc072710 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:19.239 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.231+0000 7fcbe3ed4700 1 -- 172.21.15.152:0/249389745 shutdown_connections 2024-04-02T04:55:19.239 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.231+0000 7fcbe3ed4700 1 -- 172.21.15.152:0/249389745 wait complete. 2024-04-02T04:55:19.239 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.231+0000 7fcbe3ed4700 1 Processor -- start 2024-04-02T04:55:19.240 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.232+0000 7fcbe3ed4700 1 -- start start 2024-04-02T04:55:19.240 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.232+0000 7fcbe3ed4700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcbdc0fb3b0 0x7fcbdc0fc590 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:19.240 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.232+0000 7fcbe3ed4700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcbdc102f70 0x7fcbdc0fcad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:19.240 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.232+0000 7fcbe3ed4700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcbdc0fe4e0 con 0x7fcbdc0fb3b0 2024-04-02T04:55:19.240 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.232+0000 7fcbe3ed4700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcbdc0fd010 con 0x7fcbdc102f70 2024-04-02T04:55:19.240 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.232+0000 7fcbe1c70700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcbdc0fb3b0 0x7fcbdc0fc590 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:19.241 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.232+0000 7fcbe1c70700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcbdc0fb3b0 0x7fcbdc0fc590 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:45844/0 (socket says 172.21.15.152:45844) 2024-04-02T04:55:19.241 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.232+0000 7fcbe1c70700 1 -- 172.21.15.152:0/95321789 learned_addr learned my addr 172.21.15.152:0/95321789 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:55:19.241 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.232+0000 7fcbe146f700 1 --2- 172.21.15.152:0/95321789 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcbdc102f70 0x7fcbdc0fcad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:19.241 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.232+0000 7fcbe1c70700 1 -- 172.21.15.152:0/95321789 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcbdc102f70 msgr2=0x7fcbdc0fcad0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:19.241 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.232+0000 7fcbe1c70700 1 --2- 172.21.15.152:0/95321789 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcbdc102f70 0x7fcbdc0fcad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:19.241 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.232+0000 7fcbe1c70700 1 -- 172.21.15.152:0/95321789 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcbcc005040 con 0x7fcbdc0fb3b0 2024-04-02T04:55:19.241 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.232+0000 7fcbe1c70700 1 --2- 172.21.15.152:0/95321789 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcbdc0fb3b0 0x7fcbdc0fc590 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7fcbd800eb40 tx=0x7fcbd800b040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:19.242 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.232+0000 7fcbd27fc700 1 -- 172.21.15.152:0/95321789 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fcbd8007cc0 con 0x7fcbdc0fb3b0 2024-04-02T04:55:19.242 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.232+0000 7fcbe3ed4700 1 -- 172.21.15.152:0/95321789 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcbdc0fd290 con 0x7fcbdc0fb3b0 2024-04-02T04:55:19.242 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.232+0000 7fcbd27fc700 1 -- 172.21.15.152:0/95321789 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fcbd800f040 con 0x7fcbdc0fb3b0 2024-04-02T04:55:19.242 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.232+0000 7fcbd27fc700 1 -- 172.21.15.152:0/95321789 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fcbd8009620 con 0x7fcbdc0fb3b0 2024-04-02T04:55:19.242 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.233+0000 7fcbe3ed4700 1 -- 172.21.15.152:0/95321789 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcbdc11a030 con 0x7fcbdc0fb3b0 2024-04-02T04:55:19.242 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.233+0000 7fcbe3ed4700 1 -- 172.21.15.152:0/95321789 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcbdc105350 con 0x7fcbdc0fb3b0 2024-04-02T04:55:19.242 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.234+0000 7fcbd27fc700 1 -- 172.21.15.152:0/95321789 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7fcbd8009780 con 0x7fcbdc0fb3b0 2024-04-02T04:55:19.243 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.234+0000 7fcbd27fc700 1 --2- 172.21.15.152:0/95321789 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fcbc80692a0 0x7fcbc806b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:19.243 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.234+0000 7fcbe146f700 1 --2- 172.21.15.152:0/95321789 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fcbc80692a0 0x7fcbc806b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:19.243 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.234+0000 7fcbd27fc700 1 -- 172.21.15.152:0/95321789 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7fcbd8086ad0 con 0x7fcbdc0fb3b0 2024-04-02T04:55:19.243 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.235+0000 7fcbe146f700 1 --2- 172.21.15.152:0/95321789 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fcbc80692a0 0x7fcbc806b750 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7fcbcc0043d0 tx=0x7fcbcc009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:19.247 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.243+0000 7fcbd27fc700 1 -- 172.21.15.152:0/95321789 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fcbd804e190 con 0x7fcbdc0fb3b0 2024-04-02T04:55:19.360 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.134+0000 7fa048a6b700 1 -- 172.21.15.152:0/3100399582 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fa044102e70 msgr2=0x7fa044103240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:19.360 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.134+0000 7fa048a6b700 1 --2- 172.21.15.152:0/3100399582 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fa044102e70 0x7fa044103240 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 crypto rx=0x7fa034004940 tx=0x7fa034009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:19.360 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.134+0000 7fa048a6b700 1 -- 172.21.15.152:0/3100399582 shutdown_connections 2024-04-02T04:55:19.360 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.134+0000 7fa048a6b700 1 --2- 172.21.15.152:0/3100399582 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa0440fb4a0 0x7fa0440fb910 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:19.360 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.134+0000 7fa048a6b700 1 --2- 172.21.15.152:0/3100399582 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fa044102e70 0x7fa044103240 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:19.360 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.134+0000 7fa048a6b700 1 -- 172.21.15.152:0/3100399582 >> 172.21.15.152:0/3100399582 conn(0x7fa0440714c0 msgr2=0x7fa0440718c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:19.360 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.134+0000 7fa048a6b700 1 -- 172.21.15.152:0/3100399582 shutdown_connections 2024-04-02T04:55:19.360 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.134+0000 7fa048a6b700 1 -- 172.21.15.152:0/3100399582 wait complete. 2024-04-02T04:55:19.360 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.135+0000 7fa048a6b700 1 Processor -- start 2024-04-02T04:55:19.360 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.135+0000 7fa048a6b700 1 -- start start 2024-04-02T04:55:19.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.135+0000 7fa048a6b700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fa0440fb4a0 0x7fa044113b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:19.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.135+0000 7fa048a6b700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa044102e70 0x7fa044114090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:19.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.135+0000 7fa048a6b700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa044114770 con 0x7fa0440fb4a0 2024-04-02T04:55:19.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.135+0000 7fa048a6b700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa04410dad0 con 0x7fa044102e70 2024-04-02T04:55:19.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.135+0000 7fa04259c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fa0440fb4a0 0x7fa044113b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:19.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.135+0000 7fa04259c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fa0440fb4a0 0x7fa044113b50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:45826/0 (socket says 172.21.15.152:45826) 2024-04-02T04:55:19.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.135+0000 7fa04259c700 1 -- 172.21.15.152:0/3916237895 learned_addr learned my addr 172.21.15.152:0/3916237895 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:55:19.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.135+0000 7fa041d9b700 1 --2- 172.21.15.152:0/3916237895 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa044102e70 0x7fa044114090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:19.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.135+0000 7fa04259c700 1 -- 172.21.15.152:0/3916237895 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa044102e70 msgr2=0x7fa044114090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:19.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.135+0000 7fa04259c700 1 --2- 172.21.15.152:0/3916237895 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa044102e70 0x7fa044114090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:19.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.135+0000 7fa04259c700 1 -- 172.21.15.152:0/3916237895 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa034005040 con 0x7fa0440fb4a0 2024-04-02T04:55:19.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.135+0000 7fa04259c700 1 --2- 172.21.15.152:0/3916237895 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fa0440fb4a0 0x7fa044113b50 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7fa034009150 tx=0x7fa034009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:19.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.135+0000 7fa032ffd700 1 -- 172.21.15.152:0/3916237895 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fa03400e040 con 0x7fa0440fb4a0 2024-04-02T04:55:19.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.136+0000 7fa032ffd700 1 -- 172.21.15.152:0/3916237895 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fa034013420 con 0x7fa0440fb4a0 2024-04-02T04:55:19.362 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.136+0000 7fa048a6b700 1 -- 172.21.15.152:0/3916237895 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa04410dd50 con 0x7fa0440fb4a0 2024-04-02T04:55:19.362 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.136+0000 7fa032ffd700 1 -- 172.21.15.152:0/3916237895 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fa03401c630 con 0x7fa0440fb4a0 2024-04-02T04:55:19.362 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.136+0000 7fa048a6b700 1 -- 172.21.15.152:0/3916237895 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa04410e310 con 0x7fa0440fb4a0 2024-04-02T04:55:19.362 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.137+0000 7fa048a6b700 1 -- 172.21.15.152:0/3916237895 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa044100110 con 0x7fa0440fb4a0 2024-04-02T04:55:19.362 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.137+0000 7fa032ffd700 1 -- 172.21.15.152:0/3916237895 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7fa034013590 con 0x7fa0440fb4a0 2024-04-02T04:55:19.362 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.138+0000 7fa032ffd700 1 --2- 172.21.15.152:0/3916237895 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fa02c0692b0 0x7fa02c06b760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:19.362 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.138+0000 7fa032ffd700 1 -- 172.21.15.152:0/3916237895 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7fa034088020 con 0x7fa0440fb4a0 2024-04-02T04:55:19.362 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.138+0000 7fa041d9b700 1 --2- 172.21.15.152:0/3916237895 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fa02c0692b0 0x7fa02c06b760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:19.362 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.138+0000 7fa041d9b700 1 --2- 172.21.15.152:0/3916237895 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fa02c0692b0 0x7fa02c06b760 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7fa038006fd0 tx=0x7fa038008040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:19.362 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.142+0000 7fa032ffd700 1 -- 172.21.15.152:0/3916237895 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fa0340581c0 con 0x7fa0440fb4a0 2024-04-02T04:55:19.362 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:19 smithi152 ceph-mon[29073]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:19.374 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.370+0000 7fa048a6b700 1 -- 172.21.15.152:0/3916237895 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 0} v 0) v1 -- 0x7fa04410f1a0 con 0x7fa0440fb4a0 2024-04-02T04:55:19.374 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.370+0000 7fa032ffd700 1 -- 172.21.15.152:0/3916237895 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 0}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7fa034057d50 con 0x7fa0440fb4a0 2024-04-02T04:55:19.374 INFO:teuthology.orchestra.run.smithi152.stdout:38654705702 2024-04-02T04:55:19.376 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.373+0000 7fa048a6b700 1 -- 172.21.15.152:0/3916237895 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fa02c0692b0 msgr2=0x7fa02c06b760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:19.377 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.373+0000 7fa048a6b700 1 --2- 172.21.15.152:0/3916237895 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fa02c0692b0 0x7fa02c06b760 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 crypto rx=0x7fa038006fd0 tx=0x7fa038008040 comp rx=0 tx=0).stop 2024-04-02T04:55:19.377 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.373+0000 7fa048a6b700 1 -- 172.21.15.152:0/3916237895 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fa0440fb4a0 msgr2=0x7fa044113b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:19.377 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.373+0000 7fa048a6b700 1 --2- 172.21.15.152:0/3916237895 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fa0440fb4a0 0x7fa044113b50 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 crypto rx=0x7fa034009150 tx=0x7fa034009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:19.377 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.373+0000 7fa048a6b700 1 -- 172.21.15.152:0/3916237895 shutdown_connections 2024-04-02T04:55:19.377 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.373+0000 7fa048a6b700 1 --2- 172.21.15.152:0/3916237895 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fa02c0692b0 0x7fa02c06b760 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:19.377 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.373+0000 7fa048a6b700 1 --2- 172.21.15.152:0/3916237895 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fa0440fb4a0 0x7fa044113b50 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:19.377 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.373+0000 7fa048a6b700 1 --2- 172.21.15.152:0/3916237895 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa044102e70 0x7fa044114090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:19.377 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.373+0000 7fa048a6b700 1 -- 172.21.15.152:0/3916237895 >> 172.21.15.152:0/3916237895 conn(0x7fa0440714c0 msgr2=0x7fa0440fa6c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:19.377 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.373+0000 7fa048a6b700 1 -- 172.21.15.152:0/3916237895 shutdown_connections 2024-04-02T04:55:19.377 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.373+0000 7fa048a6b700 1 -- 172.21.15.152:0/3916237895 wait complete. 2024-04-02T04:55:19.455 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.451+0000 7fcbe3ed4700 1 -- 172.21.15.152:0/95321789 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 1} v 0) v1 -- 0x7fcbdc0fda70 con 0x7fcbdc0fb3b0 2024-04-02T04:55:19.455 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.451+0000 7fcbd27fc700 1 -- 172.21.15.152:0/95321789 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 1}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7fcbd8053170 con 0x7fcbdc0fb3b0 2024-04-02T04:55:19.455 INFO:teuthology.orchestra.run.smithi152.stdout:60129542176 2024-04-02T04:55:19.457 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.453+0000 7fcbe3ed4700 1 -- 172.21.15.152:0/95321789 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fcbc80692a0 msgr2=0x7fcbc806b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:19.457 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.453+0000 7fcbe3ed4700 1 --2- 172.21.15.152:0/95321789 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fcbc80692a0 0x7fcbc806b750 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 crypto rx=0x7fcbcc0043d0 tx=0x7fcbcc009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:19.457 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.453+0000 7fcbe3ed4700 1 -- 172.21.15.152:0/95321789 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcbdc0fb3b0 msgr2=0x7fcbdc0fc590 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:19.457 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.453+0000 7fcbe3ed4700 1 --2- 172.21.15.152:0/95321789 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcbdc0fb3b0 0x7fcbdc0fc590 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 crypto rx=0x7fcbd800eb40 tx=0x7fcbd800b040 comp rx=0 tx=0).stop 2024-04-02T04:55:19.457 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.453+0000 7fcbe3ed4700 1 -- 172.21.15.152:0/95321789 shutdown_connections 2024-04-02T04:55:19.457 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.453+0000 7fcbe3ed4700 1 --2- 172.21.15.152:0/95321789 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fcbc80692a0 0x7fcbc806b750 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:19.457 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.453+0000 7fcbe3ed4700 1 --2- 172.21.15.152:0/95321789 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcbdc0fb3b0 0x7fcbdc0fc590 unknown :-1 s=CLOSED pgs=212 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:19.457 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.453+0000 7fcbe3ed4700 1 --2- 172.21.15.152:0/95321789 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcbdc102f70 0x7fcbdc0fcad0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:19.457 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.453+0000 7fcbe3ed4700 1 -- 172.21.15.152:0/95321789 >> 172.21.15.152:0/95321789 conn(0x7fcbdc072310 msgr2=0x7fcbdc0f9620 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:19.457 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.453+0000 7fcbe3ed4700 1 -- 172.21.15.152:0/95321789 shutdown_connections 2024-04-02T04:55:19.457 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:19.453+0000 7fcbe3ed4700 1 -- 172.21.15.152:0/95321789 wait complete. 2024-04-02T04:55:19.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:19 smithi154 ceph-mon[35729]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:20.259 INFO:tasks.cephadm.ceph_manager.ceph:need seq 60129542173 got 60129542176 for osd.1 2024-04-02T04:55:20.259 DEBUG:teuthology.parallel:result is None 2024-04-02T04:55:20.385 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:20 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/3916237895' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-02T04:55:20.385 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:20 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/95321789' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-02T04:55:20.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:20 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/3916237895' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-02T04:55:20.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:20 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/95321789' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-02T04:55:20.486 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.482+0000 7f62eed58700 1 -- 172.21.15.152:0/695796552 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f62e8102e10 msgr2=0x7f62e81031e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:20.486 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.482+0000 7f62eed58700 1 --2- 172.21.15.152:0/695796552 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f62e8102e10 0x7f62e81031e0 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 crypto rx=0x7f62dc004940 tx=0x7f62dc009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:20.487 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.482+0000 7f62eed58700 1 -- 172.21.15.152:0/695796552 shutdown_connections 2024-04-02T04:55:20.487 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.482+0000 7f62eed58700 1 --2- 172.21.15.152:0/695796552 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f62e81037b0 0x7f62e80fce20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:20.487 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.482+0000 7f62eed58700 1 --2- 172.21.15.152:0/695796552 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f62e8102e10 0x7f62e81031e0 unknown :-1 s=CLOSED pgs=213 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:20.487 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.482+0000 7f62eed58700 1 -- 172.21.15.152:0/695796552 >> 172.21.15.152:0/695796552 conn(0x7f62e80f8980 msgr2=0x7f62e80fad90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:20.487 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.482+0000 7f62eed58700 1 -- 172.21.15.152:0/695796552 shutdown_connections 2024-04-02T04:55:20.487 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.482+0000 7f62eed58700 1 -- 172.21.15.152:0/695796552 wait complete. 2024-04-02T04:55:20.487 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.483+0000 7f62eed58700 1 Processor -- start 2024-04-02T04:55:20.487 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.483+0000 7f62eed58700 1 -- start start 2024-04-02T04:55:20.489 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.483+0000 7f62eed58700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f62e8102e10 0x7f62e8115d80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:20.489 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.483+0000 7f62eed58700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f62e81037b0 0x7f62e81162c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:20.489 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.483+0000 7f62eed58700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f62e81169a0 con 0x7f62e81037b0 2024-04-02T04:55:20.489 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.483+0000 7f62eed58700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f62e810fce0 con 0x7f62e8102e10 2024-04-02T04:55:20.489 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.483+0000 7f62e7fff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f62e81037b0 0x7f62e81162c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:20.489 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.483+0000 7f62e7fff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f62e81037b0 0x7f62e81162c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:45862/0 (socket says 172.21.15.152:45862) 2024-04-02T04:55:20.489 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.483+0000 7f62e7fff700 1 -- 172.21.15.152:0/644459785 learned_addr learned my addr 172.21.15.152:0/644459785 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:55:20.489 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.484+0000 7f62ecaf4700 1 --2- 172.21.15.152:0/644459785 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f62e8102e10 0x7f62e8115d80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:20.490 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.484+0000 7f62e7fff700 1 -- 172.21.15.152:0/644459785 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f62e8102e10 msgr2=0x7f62e8115d80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:20.490 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.484+0000 7f62e7fff700 1 --2- 172.21.15.152:0/644459785 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f62e8102e10 0x7f62e8115d80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:20.490 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.484+0000 7f62e7fff700 1 -- 172.21.15.152:0/644459785 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f62dc005040 con 0x7f62e81037b0 2024-04-02T04:55:20.490 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.484+0000 7f62e7fff700 1 --2- 172.21.15.152:0/644459785 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f62e81037b0 0x7f62e81162c0 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7f62d400b790 tx=0x7f62d4009220 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:20.490 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.484+0000 7f62e57fa700 1 -- 172.21.15.152:0/644459785 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f62d400f6f0 con 0x7f62e81037b0 2024-04-02T04:55:20.490 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.484+0000 7f62e57fa700 1 -- 172.21.15.152:0/644459785 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f62d400fd20 con 0x7f62e81037b0 2024-04-02T04:55:20.490 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.484+0000 7f62eed58700 1 -- 172.21.15.152:0/644459785 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f62e810ff60 con 0x7f62e81037b0 2024-04-02T04:55:20.490 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.484+0000 7f62e57fa700 1 -- 172.21.15.152:0/644459785 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f62d40103a0 con 0x7f62e81037b0 2024-04-02T04:55:20.490 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.484+0000 7f62eed58700 1 -- 172.21.15.152:0/644459785 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f62e8110520 con 0x7f62e81037b0 2024-04-02T04:55:20.490 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.485+0000 7f62eed58700 1 -- 172.21.15.152:0/644459785 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f62e80fe560 con 0x7f62e81037b0 2024-04-02T04:55:20.490 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.485+0000 7f62e57fa700 1 -- 172.21.15.152:0/644459785 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f62d4011400 con 0x7f62e81037b0 2024-04-02T04:55:20.491 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.486+0000 7f62e57fa700 1 --2- 172.21.15.152:0/644459785 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f62d80692a0 0x7f62d806b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:20.491 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.486+0000 7f62e57fa700 1 -- 172.21.15.152:0/644459785 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7f62d4086c20 con 0x7f62e81037b0 2024-04-02T04:55:20.491 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.486+0000 7f62ecaf4700 1 --2- 172.21.15.152:0/644459785 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f62d80692a0 0x7f62d806b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:20.492 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.486+0000 7f62ecaf4700 1 --2- 172.21.15.152:0/644459785 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f62d80692a0 0x7f62d806b750 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f62dc004f90 tx=0x7f62dc009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:20.494 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.490+0000 7f62e57fa700 1 -- 172.21.15.152:0/644459785 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f62d4056d20 con 0x7f62e81037b0 2024-04-02T04:55:20.681 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.677+0000 7f62eed58700 1 -- 172.21.15.152:0/644459785 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "osd last-stat-seq", "id": 2} v 0) v1 -- 0x7f62e804bed0 con 0x7f62e81037b0 2024-04-02T04:55:20.681 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.678+0000 7f62e57fa700 1 -- 172.21.15.152:0/644459785 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "osd last-stat-seq", "id": 2}]=0 v0) v1 ==== 74+0+12 (secure 0 0 0) 0x7f62d40568b0 con 0x7f62e81037b0 2024-04-02T04:55:20.682 INFO:teuthology.orchestra.run.smithi152.stdout:81604378652 2024-04-02T04:55:20.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.680+0000 7f62eed58700 1 -- 172.21.15.152:0/644459785 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f62d80692a0 msgr2=0x7f62d806b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:20.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.680+0000 7f62eed58700 1 --2- 172.21.15.152:0/644459785 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f62d80692a0 0x7f62d806b750 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 crypto rx=0x7f62dc004f90 tx=0x7f62dc009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:20.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.680+0000 7f62eed58700 1 -- 172.21.15.152:0/644459785 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f62e81037b0 msgr2=0x7f62e81162c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:20.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.680+0000 7f62eed58700 1 --2- 172.21.15.152:0/644459785 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f62e81037b0 0x7f62e81162c0 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 crypto rx=0x7f62d400b790 tx=0x7f62d4009220 comp rx=0 tx=0).stop 2024-04-02T04:55:20.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.680+0000 7f62eed58700 1 -- 172.21.15.152:0/644459785 shutdown_connections 2024-04-02T04:55:20.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.680+0000 7f62eed58700 1 --2- 172.21.15.152:0/644459785 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f62d80692a0 0x7f62d806b750 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:20.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.680+0000 7f62eed58700 1 --2- 172.21.15.152:0/644459785 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f62e8102e10 0x7f62e8115d80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:20.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.680+0000 7f62eed58700 1 --2- 172.21.15.152:0/644459785 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f62e81037b0 0x7f62e81162c0 unknown :-1 s=CLOSED pgs=214 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:20.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.680+0000 7f62eed58700 1 -- 172.21.15.152:0/644459785 >> 172.21.15.152:0/644459785 conn(0x7f62e80f8980 msgr2=0x7f62e8071be0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:20.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.680+0000 7f62eed58700 1 -- 172.21.15.152:0/644459785 shutdown_connections 2024-04-02T04:55:20.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:20.680+0000 7f62eed58700 1 -- 172.21.15.152:0/644459785 wait complete. 2024-04-02T04:55:21.096 INFO:tasks.cephadm.ceph_manager.ceph:need seq 38654705700 got 38654705702 for osd.0 2024-04-02T04:55:21.096 DEBUG:teuthology.parallel:result is None 2024-04-02T04:55:21.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:21 smithi154 ceph-mon[35729]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:21.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:21 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/644459785' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-02T04:55:21.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:21 smithi152 ceph-mon[29073]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:21.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:21 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/644459785' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-02T04:55:21.858 INFO:tasks.cephadm.ceph_manager.ceph:need seq 81604378648 got 81604378652 for osd.2 2024-04-02T04:55:21.859 DEBUG:teuthology.parallel:result is None 2024-04-02T04:55:21.859 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-02T04:55:21.859 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-02T04:55:22.122 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:55:23.225 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:23 smithi152 ceph-mon[29073]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:23.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:23 smithi154 ceph-mon[35729]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:23.779 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.774+0000 7feea4c59700 1 -- 172.21.15.152:0/2314424827 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7feea0101d70 msgr2=0x7feea0102140 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:23.780 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.774+0000 7feea4c59700 1 --2- 172.21.15.152:0/2314424827 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7feea0101d70 0x7feea0102140 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 crypto rx=0x7fee94004940 tx=0x7fee94009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:23.780 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.774+0000 7feea4c59700 1 -- 172.21.15.152:0/2314424827 shutdown_connections 2024-04-02T04:55:23.780 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.774+0000 7feea4c59700 1 --2- 172.21.15.152:0/2314424827 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feea0102710 0x7feea00fbd80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:23.780 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.774+0000 7feea4c59700 1 --2- 172.21.15.152:0/2314424827 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7feea0101d70 0x7feea0102140 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:23.780 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.774+0000 7feea4c59700 1 -- 172.21.15.152:0/2314424827 >> 172.21.15.152:0/2314424827 conn(0x7feea0072bc0 msgr2=0x7feea0072fc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:23.780 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.774+0000 7feea4c59700 1 -- 172.21.15.152:0/2314424827 shutdown_connections 2024-04-02T04:55:23.780 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.774+0000 7feea4c59700 1 -- 172.21.15.152:0/2314424827 wait complete. 2024-04-02T04:55:23.780 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.775+0000 7feea4c59700 1 Processor -- start 2024-04-02T04:55:23.780 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.775+0000 7feea4c59700 1 -- start start 2024-04-02T04:55:23.780 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.775+0000 7feea4c59700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7feea0101d70 0x7feea010eae0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:23.780 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.775+0000 7feea4c59700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feea0102710 0x7feea010f020 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:23.780 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.775+0000 7feea4c59700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feea010f670 con 0x7feea0101d70 2024-04-02T04:55:23.781 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.775+0000 7feea4c59700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feea010f7b0 con 0x7feea0102710 2024-04-02T04:55:23.781 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.775+0000 7fee9e59c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7feea0101d70 0x7feea010eae0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:23.781 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.775+0000 7fee9e59c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7feea0101d70 0x7feea010eae0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:46268/0 (socket says 172.21.15.152:46268) 2024-04-02T04:55:23.781 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.775+0000 7fee9e59c700 1 -- 172.21.15.152:0/3908820438 learned_addr learned my addr 172.21.15.152:0/3908820438 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:55:23.781 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.775+0000 7fee9dd9b700 1 --2- 172.21.15.152:0/3908820438 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feea0102710 0x7feea010f020 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:23.781 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.775+0000 7fee9e59c700 1 -- 172.21.15.152:0/3908820438 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feea0102710 msgr2=0x7feea010f020 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:23.781 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.775+0000 7fee9e59c700 1 --2- 172.21.15.152:0/3908820438 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feea0102710 0x7feea010f020 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:23.781 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.775+0000 7fee9e59c700 1 -- 172.21.15.152:0/3908820438 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fee94005040 con 0x7feea0101d70 2024-04-02T04:55:23.781 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.775+0000 7fee9dd9b700 1 --2- 172.21.15.152:0/3908820438 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feea0102710 0x7feea010f020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-02T04:55:23.781 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.776+0000 7fee9e59c700 1 --2- 172.21.15.152:0/3908820438 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7feea0101d70 0x7feea010eae0 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7fee9400b8b0 tx=0x7fee94009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:23.781 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.776+0000 7fee8effd700 1 -- 172.21.15.152:0/3908820438 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fee9400e040 con 0x7feea0101d70 2024-04-02T04:55:23.781 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.776+0000 7feea4c59700 1 -- 172.21.15.152:0/3908820438 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7feea0117b50 con 0x7feea0101d70 2024-04-02T04:55:23.781 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.776+0000 7fee8effd700 1 -- 172.21.15.152:0/3908820438 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fee94004290 con 0x7feea0101d70 2024-04-02T04:55:23.781 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.776+0000 7fee8effd700 1 -- 172.21.15.152:0/3908820438 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fee9401c3f0 con 0x7feea0101d70 2024-04-02T04:55:23.781 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.776+0000 7feea4c59700 1 -- 172.21.15.152:0/3908820438 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7feea0118060 con 0x7feea0101d70 2024-04-02T04:55:23.782 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.777+0000 7feea4c59700 1 -- 172.21.15.152:0/3908820438 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7feea00ff080 con 0x7feea0101d70 2024-04-02T04:55:23.782 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.777+0000 7fee8effd700 1 -- 172.21.15.152:0/3908820438 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7fee940137b0 con 0x7feea0101d70 2024-04-02T04:55:23.782 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.778+0000 7fee8effd700 1 --2- 172.21.15.152:0/3908820438 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fee880692a0 0x7fee8806b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:23.782 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.778+0000 7fee8effd700 1 -- 172.21.15.152:0/3908820438 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7fee940881b0 con 0x7feea0101d70 2024-04-02T04:55:23.783 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.778+0000 7fee9dd9b700 1 --2- 172.21.15.152:0/3908820438 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fee880692a0 0x7fee8806b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:23.783 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.778+0000 7fee9dd9b700 1 --2- 172.21.15.152:0/3908820438 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fee880692a0 0x7fee8806b750 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7fee90007fd0 tx=0x7fee90009040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:23.785 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.782+0000 7fee8effd700 1 -- 172.21.15.152:0/3908820438 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fee94054820 con 0x7feea0101d70 2024-04-02T04:55:23.966 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.962+0000 7feea4c59700 1 -- 172.21.15.152:0/3908820438 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7feea0108ee0 con 0x7fee880692a0 2024-04-02T04:55:23.968 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.964+0000 7fee8effd700 1 -- 172.21.15.152:0/3908820438 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+27807 (secure 0 0 0) 0x7feea0108ee0 con 0x7fee880692a0 2024-04-02T04:55:23.968 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:55:23.970 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.966+0000 7feea4c59700 1 -- 172.21.15.152:0/3908820438 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fee880692a0 msgr2=0x7fee8806b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:23.970 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.966+0000 7feea4c59700 1 --2- 172.21.15.152:0/3908820438 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fee880692a0 0x7fee8806b750 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 crypto rx=0x7fee90007fd0 tx=0x7fee90009040 comp rx=0 tx=0).stop 2024-04-02T04:55:23.970 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.966+0000 7feea4c59700 1 -- 172.21.15.152:0/3908820438 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7feea0101d70 msgr2=0x7feea010eae0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:23.970 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.966+0000 7feea4c59700 1 --2- 172.21.15.152:0/3908820438 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7feea0101d70 0x7feea010eae0 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 crypto rx=0x7fee9400b8b0 tx=0x7fee94009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:23.970 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.966+0000 7feea4c59700 1 -- 172.21.15.152:0/3908820438 shutdown_connections 2024-04-02T04:55:23.970 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.966+0000 7feea4c59700 1 --2- 172.21.15.152:0/3908820438 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fee880692a0 0x7fee8806b750 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:23.970 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.966+0000 7feea4c59700 1 --2- 172.21.15.152:0/3908820438 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7feea0101d70 0x7feea010eae0 unknown :-1 s=CLOSED pgs=216 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:23.970 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.966+0000 7feea4c59700 1 --2- 172.21.15.152:0/3908820438 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7feea0102710 0x7feea010f020 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:23.970 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.966+0000 7feea4c59700 1 -- 172.21.15.152:0/3908820438 >> 172.21.15.152:0/3908820438 conn(0x7feea0072bc0 msgr2=0x7feea00fafe0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:23.970 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.966+0000 7feea4c59700 1 -- 172.21.15.152:0/3908820438 shutdown_connections 2024-04-02T04:55:23.971 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:23.966+0000 7feea4c59700 1 -- 172.21.15.152:0/3908820438 wait complete. 2024-04-02T04:55:23.971 INFO:teuthology.orchestra.run.smithi152.stderr:dumped all 2024-04-02T04:55:24.693 INFO:teuthology.orchestra.run.smithi152.stdout:{"pg_ready":true,"pg_map":{"version":154,"stamp":"2024-04-02T04:55:21.996279+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":3,"kb":562470912,"kb_used":1788036,"kb_used_data":3876,"kb_used_omap":9,"kb_used_meta":1784054,"kb_avail":560682876,"statfs":{"total":575970213888,"available":574139265024,"internally_reserved":0,"allocated":3969024,"data_stored":2606262,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10224,"internal_metadata":1826871312},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002317"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":189,"reported_epoch":39,"state":"active+clean","last_fresh":"2024-04-02T04:54:32.122183+0000","last_change":"2024-04-02T04:54:06.839975+0000","last_active":"2024-04-02T04:54:32.122183+0000","last_peered":"2024-04-02T04:54:32.122183+0000","last_clean":"2024-04-02T04:54:32.122183+0000","last_became_active":"2024-04-02T04:54:06.839577+0000","last_became_peered":"2024-04-02T04:54:06.839577+0000","last_unstale":"2024-04-02T04:54:32.122183+0000","last_undegraded":"2024-04-02T04:54:32.122183+0000","last_fullsized":"2024-04-02T04:54:32.122183+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-02T04:53:20.444528+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-02T04:53:20.444528+0000","last_clean_scrub_stamp":"2024-04-02T04:53:20.444528+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-03T08:25:58.858132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":5,"up_from":39,"seq":167503724556,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":297712,"kb_used_data":356,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":364544,"data_stored":139193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.443}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46700000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49199999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33000000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45700000000000002}]}]},{"osd":4,"up_from":34,"seq":146028888082,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":297716,"kb_used_data":356,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":364544,"data_stored":139193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 04:55:10 2024","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.46999999999999997}]},{"osd":1,"last update":"Tue Apr 2 04:55:10 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.45000000000000001}]},{"osd":2,"last update":"Tue Apr 2 04:55:10 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.50900000000000001}]},{"osd":3,"last update":"Tue Apr 2 04:55:10 2024","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.375,"15min":0.375},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"last":0.378},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.36199999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33900000000000002}]}]},{"osd":3,"up_from":26,"seq":111669149718,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298300,"kb_used_data":936,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446852,"statfs":{"total":95995035648,"available":95689576448,"internally_reserved":0,"allocated":958464,"data_stored":729561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 04:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.505},{"interface":"front","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.436}]},{"osd":1,"last update":"Tue Apr 2 04:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.39800000000000002}]},{"osd":2,"last update":"Tue Apr 2 04:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.44700000000000001}]},{"osd":4,"last update":"Tue Apr 2 04:55:10 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.308},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.34200000000000003}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41699999999999998}]}]},{"osd":2,"up_from":19,"seq":81604378652,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":297716,"kb_used_data":356,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":364544,"data_stored":139193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 04:54:18 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"last":0.436}]},{"osd":1,"last update":"Tue Apr 2 04:54:18 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.38,"15min":0.38},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.498,"5min":0.498,"15min":0.498},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"last":0.45000000000000001}]},{"osd":3,"last update":"Tue Apr 2 04:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.72799999999999998}]},{"osd":4,"last update":"Tue Apr 2 04:55:09 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.71499999999999997}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64200000000000002}]}]},{"osd":0,"up_from":9,"seq":38654705703,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298296,"kb_used_data":936,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446856,"statfs":{"total":95995035648,"available":95689580544,"internally_reserved":0,"allocated":958464,"data_stored":729561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Apr 2 04:54:54 2024","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.32800000000000001,"15min":0.32200000000000001},"min":{"1min":0.223,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.34899999999999998,"15min":0.34699999999999998},"min":{"1min":0.23499999999999999,"5min":0.216,"15min":0.216},"max":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.46700000000000003}]},{"osd":2,"last update":"Tue Apr 2 04:54:20 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.45000000000000001}]},{"osd":3,"last update":"Tue Apr 2 04:54:45 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.61099999999999999}]},{"osd":4,"last update":"Tue Apr 2 04:55:07 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.626},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.64100000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59099999999999997}]}]},{"osd":1,"up_from":14,"seq":60129542177,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298296,"kb_used_data":936,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446856,"statfs":{"total":95995035648,"available":95689580544,"internally_reserved":0,"allocated":958464,"data_stored":729561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 04:54:53 2024","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.33200000000000002,"15min":0.32700000000000001},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.34100000000000003,"15min":0.33400000000000002},"min":{"1min":0.252,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"last":0.32100000000000001}]},{"osd":2,"last update":"Tue Apr 2 04:54:20 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"last":0.307}]},{"osd":3,"last update":"Tue Apr 2 04:54:45 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.499},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.42499999999999999}]},{"osd":4,"last update":"Tue Apr 2 04:55:07 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.45700000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47399999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-02T04:55:24.694 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph pg dump --format=json 2024-04-02T04:55:24.951 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:55:25.405 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:25 smithi152 ceph-mon[29073]: from='client.14416 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T04:55:25.405 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:25 smithi152 ceph-mon[29073]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:25.405 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:25 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:55:25.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:25 smithi154 ceph-mon[35729]: from='client.14416 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T04:55:25.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:25 smithi154 ceph-mon[35729]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:25.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:25 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:55:26.460 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.455+0000 7f61acce8700 1 -- 172.21.15.152:0/381677396 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61a8101d20 msgr2=0x7f61a81020f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:26.461 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.455+0000 7f61acce8700 1 --2- 172.21.15.152:0/381677396 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61a8101d20 0x7f61a81020f0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 crypto rx=0x7f6198004940 tx=0x7f6198009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:26.461 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.455+0000 7f61acce8700 1 -- 172.21.15.152:0/381677396 shutdown_connections 2024-04-02T04:55:26.461 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.455+0000 7f61acce8700 1 --2- 172.21.15.152:0/381677396 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f61a81026c0 0x7f61a80fbd30 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:26.461 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.455+0000 7f61acce8700 1 --2- 172.21.15.152:0/381677396 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61a8101d20 0x7f61a81020f0 unknown :-1 s=CLOSED pgs=217 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:26.461 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.455+0000 7f61acce8700 1 -- 172.21.15.152:0/381677396 >> 172.21.15.152:0/381677396 conn(0x7f61a80f78b0 msgr2=0x7f61a80f9cc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:26.461 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.455+0000 7f61acce8700 1 -- 172.21.15.152:0/381677396 shutdown_connections 2024-04-02T04:55:26.461 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.455+0000 7f61acce8700 1 -- 172.21.15.152:0/381677396 wait complete. 2024-04-02T04:55:26.461 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.456+0000 7f61acce8700 1 Processor -- start 2024-04-02T04:55:26.461 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.456+0000 7f61acce8700 1 -- start start 2024-04-02T04:55:26.461 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.456+0000 7f61acce8700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f61a8101d20 0x7f61a810eb40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:26.461 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.456+0000 7f61acce8700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61a81026c0 0x7f61a810f080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:26.461 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.456+0000 7f61acce8700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f61a810f5c0 con 0x7f61a81026c0 2024-04-02T04:55:26.462 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.456+0000 7f61acce8700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f61a810f700 con 0x7f61a8101d20 2024-04-02T04:55:26.462 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.456+0000 7f61a5d9b700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61a81026c0 0x7f61a810f080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:26.462 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.456+0000 7f61a5d9b700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61a81026c0 0x7f61a810f080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:46288/0 (socket says 172.21.15.152:46288) 2024-04-02T04:55:26.462 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.456+0000 7f61a5d9b700 1 -- 172.21.15.152:0/2626561624 learned_addr learned my addr 172.21.15.152:0/2626561624 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:55:26.462 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.456+0000 7f61a659c700 1 --2- 172.21.15.152:0/2626561624 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f61a8101d20 0x7f61a810eb40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:26.462 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.456+0000 7f61a5d9b700 1 -- 172.21.15.152:0/2626561624 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f61a8101d20 msgr2=0x7f61a810eb40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:26.462 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.456+0000 7f61a5d9b700 1 --2- 172.21.15.152:0/2626561624 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f61a8101d20 0x7f61a810eb40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:26.462 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.456+0000 7f61a5d9b700 1 -- 172.21.15.152:0/2626561624 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6198005040 con 0x7f61a81026c0 2024-04-02T04:55:26.462 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.456+0000 7f61a5d9b700 1 --2- 172.21.15.152:0/2626561624 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61a81026c0 0x7f61a810f080 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f619c002a50 tx=0x7f619c004600 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:26.462 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.457+0000 7f6196ffd700 1 -- 172.21.15.152:0/2626561624 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f619c007cc0 con 0x7f61a81026c0 2024-04-02T04:55:26.462 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.457+0000 7f61acce8700 1 -- 172.21.15.152:0/2626561624 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f61a8117bb0 con 0x7f61a81026c0 2024-04-02T04:55:26.462 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.457+0000 7f6196ffd700 1 -- 172.21.15.152:0/2626561624 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f619c00f040 con 0x7f61a81026c0 2024-04-02T04:55:26.462 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.457+0000 7f6196ffd700 1 -- 172.21.15.152:0/2626561624 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f619c00d9c0 con 0x7f61a81026c0 2024-04-02T04:55:26.463 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.457+0000 7f61acce8700 1 -- 172.21.15.152:0/2626561624 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f61a8117f80 con 0x7f61a81026c0 2024-04-02T04:55:26.463 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.457+0000 7f61acce8700 1 -- 172.21.15.152:0/2626561624 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f61a80fd470 con 0x7f61a81026c0 2024-04-02T04:55:26.463 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.458+0000 7f6196ffd700 1 -- 172.21.15.152:0/2626561624 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f619c009730 con 0x7f61a81026c0 2024-04-02T04:55:26.463 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.458+0000 7f6196ffd700 1 --2- 172.21.15.152:0/2626561624 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f61900692a0 0x7f619006b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:26.464 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.458+0000 7f6196ffd700 1 -- 172.21.15.152:0/2626561624 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7f619c086bf0 con 0x7f61a81026c0 2024-04-02T04:55:26.464 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.458+0000 7f61a659c700 1 --2- 172.21.15.152:0/2626561624 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f61900692a0 0x7f619006b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:26.464 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.459+0000 7f61a659c700 1 --2- 172.21.15.152:0/2626561624 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f61900692a0 0x7f619006b750 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f6198004f90 tx=0x7f6198009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:26.466 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.463+0000 7f6196ffd700 1 -- 172.21.15.152:0/2626561624 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f619c0531e0 con 0x7f61a81026c0 2024-04-02T04:55:26.652 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.648+0000 7f61acce8700 1 -- 172.21.15.152:0/2626561624 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f61a81055c0 con 0x7f61900692a0 2024-04-02T04:55:26.653 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.650+0000 7f6196ffd700 1 -- 172.21.15.152:0/2626561624 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 dumped all) v1 ==== 18+0+27652 (secure 0 0 0) 0x7f61a81055c0 con 0x7f61900692a0 2024-04-02T04:55:26.654 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:55:26.656 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.652+0000 7f61acce8700 1 -- 172.21.15.152:0/2626561624 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f61900692a0 msgr2=0x7f619006b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:26.656 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.652+0000 7f61acce8700 1 --2- 172.21.15.152:0/2626561624 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f61900692a0 0x7f619006b750 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 crypto rx=0x7f6198004f90 tx=0x7f6198009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:26.656 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.652+0000 7f61acce8700 1 -- 172.21.15.152:0/2626561624 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61a81026c0 msgr2=0x7f61a810f080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:26.656 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.652+0000 7f61acce8700 1 --2- 172.21.15.152:0/2626561624 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61a81026c0 0x7f61a810f080 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 crypto rx=0x7f619c002a50 tx=0x7f619c004600 comp rx=0 tx=0).stop 2024-04-02T04:55:26.656 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.652+0000 7f61acce8700 1 -- 172.21.15.152:0/2626561624 shutdown_connections 2024-04-02T04:55:26.656 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.652+0000 7f61acce8700 1 --2- 172.21.15.152:0/2626561624 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f61900692a0 0x7f619006b750 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:26.656 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.652+0000 7f61acce8700 1 --2- 172.21.15.152:0/2626561624 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f61a8101d20 0x7f61a810eb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:26.657 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.652+0000 7f61acce8700 1 --2- 172.21.15.152:0/2626561624 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f61a81026c0 0x7f61a810f080 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:26.657 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.652+0000 7f61acce8700 1 -- 172.21.15.152:0/2626561624 >> 172.21.15.152:0/2626561624 conn(0x7f61a80f78b0 msgr2=0x7f61a8104eb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:26.657 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.652+0000 7f61acce8700 1 -- 172.21.15.152:0/2626561624 shutdown_connections 2024-04-02T04:55:26.657 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:26.652+0000 7f61acce8700 1 -- 172.21.15.152:0/2626561624 wait complete. 2024-04-02T04:55:26.657 INFO:teuthology.orchestra.run.smithi152.stderr:dumped all 2024-04-02T04:55:27.317 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:27 smithi152 ceph-mon[29073]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:27.346 INFO:teuthology.orchestra.run.smithi152.stdout:{"pg_ready":true,"pg_map":{"version":156,"stamp":"2024-04-02T04:55:25.996905+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":3,"kb":562470912,"kb_used":1788036,"kb_used_data":3876,"kb_used_omap":9,"kb_used_meta":1784054,"kb_avail":560682876,"statfs":{"total":575970213888,"available":574139265024,"internally_reserved":0,"allocated":3969024,"data_stored":2606262,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10224,"internal_metadata":1826871312},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002277"},"pg_stats":[{"pgid":"1.0","version":"23'117","reported_seq":189,"reported_epoch":39,"state":"active+clean","last_fresh":"2024-04-02T04:54:32.122183+0000","last_change":"2024-04-02T04:54:06.839975+0000","last_active":"2024-04-02T04:54:32.122183+0000","last_peered":"2024-04-02T04:54:32.122183+0000","last_clean":"2024-04-02T04:54:32.122183+0000","last_became_active":"2024-04-02T04:54:06.839577+0000","last_became_peered":"2024-04-02T04:54:06.839577+0000","last_unstale":"2024-04-02T04:54:32.122183+0000","last_undegraded":"2024-04-02T04:54:32.122183+0000","last_fullsized":"2024-04-02T04:54:32.122183+0000","mapping_epoch":33,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":34,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-02T04:53:20.444528+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-02T04:53:20.444528+0000","last_clean_scrub_stamp":"2024-04-02T04:53:20.444528+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-03T08:25:58.858132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":116,"num_read_kb":100,"num_write":159,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117,"ondisk_log_size":117,"up":3,"acting":3,"num_store_stats":4}],"osd_stats":[{"osd":5,"up_from":39,"seq":167503724557,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":297712,"kb_used_data":356,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447440,"statfs":{"total":95995035648,"available":95690178560,"internally_reserved":0,"allocated":364544,"data_stored":139193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55300000000000005}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60599999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51500000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.318}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33400000000000002}]}]},{"osd":4,"up_from":34,"seq":146028888083,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":297716,"kb_used_data":356,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":364544,"data_stored":139193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 04:55:10 2024","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.49199999999999999}]},{"osd":1,"last update":"Tue Apr 2 04:55:10 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.43099999999999999}]},{"osd":2,"last update":"Tue Apr 2 04:55:10 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.372}]},{"osd":3,"last update":"Tue Apr 2 04:55:10 2024","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.375,"15min":0.375},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.33100000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34799999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149719,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298300,"kb_used_data":936,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446852,"statfs":{"total":95995035648,"available":95689576448,"internally_reserved":0,"allocated":958464,"data_stored":729561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 04:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.629},{"interface":"front","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.52800000000000002}]},{"osd":1,"last update":"Tue Apr 2 04:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.57499999999999996}]},{"osd":2,"last update":"Tue Apr 2 04:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.59299999999999997}]},{"osd":4,"last update":"Tue Apr 2 04:55:10 2024","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.309},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.34399999999999997}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38800000000000001}]}]},{"osd":2,"up_from":19,"seq":81604378653,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":297716,"kb_used_data":356,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447436,"statfs":{"total":95995035648,"available":95690174464,"internally_reserved":0,"allocated":364544,"data_stored":139193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 04:55:21 2024","interfaces":[{"interface":"back","average":{"1min":0.30099999999999999,"5min":0.35099999999999998,"15min":0.35899999999999999},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":0.46600000000000003,"5min":0.56499999999999995,"15min":0.56499999999999995},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.308,"5min":0.34999999999999998,"15min":0.35699999999999998},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.436,"5min":0.50600000000000001,"15min":0.50600000000000001},"last":0.33100000000000002}]},{"osd":1,"last update":"Tue Apr 2 04:55:21 2024","interfaces":[{"interface":"back","average":{"1min":0.318,"5min":0.36699999999999999,"15min":0.375},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.42599999999999999,"5min":0.498,"15min":0.498},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.32900000000000001,"5min":0.37,"15min":0.377},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.45000000000000001,"5min":0.54100000000000004,"15min":0.54100000000000004},"last":0.314}]},{"osd":3,"last update":"Tue Apr 2 04:54:44 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.61899999999999999}]},{"osd":4,"last update":"Tue Apr 2 04:55:09 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.51100000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68400000000000005}]}]},{"osd":0,"up_from":9,"seq":38654705704,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298296,"kb_used_data":936,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446856,"statfs":{"total":95995035648,"available":95689580544,"internally_reserved":0,"allocated":958464,"data_stored":729561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Apr 2 04:54:54 2024","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.32800000000000001,"15min":0.32200000000000001},"min":{"1min":0.223,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.34899999999999998,"15min":0.34699999999999998},"min":{"1min":0.23499999999999999,"5min":0.216,"15min":0.216},"max":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.35099999999999998}]},{"osd":2,"last update":"Tue Apr 2 04:55:20 2024","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":1.5840000000000001,"5min":1.5840000000000001,"15min":1.5840000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.42699999999999999,"15min":0.433},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":1.504,"5min":1.504,"15min":1.504},"last":0.36799999999999999}]},{"osd":3,"last update":"Tue Apr 2 04:54:45 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.621},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.94899999999999995,"5min":0.94899999999999995,"15min":0.94899999999999995},"last":0.60099999999999998}]},{"osd":4,"last update":"Tue Apr 2 04:55:07 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.56299999999999994}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54300000000000004}]}]},{"osd":1,"up_from":14,"seq":60129542177,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298296,"kb_used_data":936,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446856,"statfs":{"total":95995035648,"available":95689580544,"internally_reserved":0,"allocated":958464,"data_stored":729561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 2 04:54:53 2024","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.33200000000000002,"15min":0.32700000000000001},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.34100000000000003,"15min":0.33400000000000002},"min":{"1min":0.252,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"last":0.32100000000000001}]},{"osd":2,"last update":"Tue Apr 2 04:54:20 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"last":0.307}]},{"osd":3,"last update":"Tue Apr 2 04:54:45 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.499},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.42499999999999999}]},{"osd":4,"last update":"Tue Apr 2 04:55:07 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.45700000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47399999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-02T04:55:27.348 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-02T04:55:27.348 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-02T04:55:27.348 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-02T04:55:27.348 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph health --format=json 2024-04-02T04:55:27.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:27 smithi154 ceph-mon[35729]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:27.607 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:55:28.408 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:28 smithi152 ceph-mon[29073]: from='client.14420 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T04:55:28.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:28 smithi154 ceph-mon[35729]: from='client.14420 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-02T04:55:29.283 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:29 smithi152 ceph-mon[29073]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:29.341 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.336+0000 7fc0c7d1e700 1 -- 172.21.15.152:0/3244948991 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc0c00fb4a0 msgr2=0x7fc0c00fb910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:29.341 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.336+0000 7fc0c7d1e700 1 --2- 172.21.15.152:0/3244948991 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc0c00fb4a0 0x7fc0c00fb910 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 crypto rx=0x7fc0b0004940 tx=0x7fc0b0009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:29.342 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.336+0000 7fc0c7d1e700 1 -- 172.21.15.152:0/3244948991 shutdown_connections 2024-04-02T04:55:29.342 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.336+0000 7fc0c7d1e700 1 --2- 172.21.15.152:0/3244948991 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc0c00fb4a0 0x7fc0c00fb910 unknown :-1 s=CLOSED pgs=219 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:29.342 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.336+0000 7fc0c7d1e700 1 --2- 172.21.15.152:0/3244948991 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc0c0102e70 0x7fc0c0103240 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:29.342 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.336+0000 7fc0c7d1e700 1 -- 172.21.15.152:0/3244948991 >> 172.21.15.152:0/3244948991 conn(0x7fc0c00714c0 msgr2=0x7fc0c00718c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:29.342 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.336+0000 7fc0c7d1e700 1 -- 172.21.15.152:0/3244948991 shutdown_connections 2024-04-02T04:55:29.342 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.336+0000 7fc0c7d1e700 1 -- 172.21.15.152:0/3244948991 wait complete. 2024-04-02T04:55:29.342 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.336+0000 7fc0c7d1e700 1 Processor -- start 2024-04-02T04:55:29.342 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.336+0000 7fc0c7d1e700 1 -- start start 2024-04-02T04:55:29.342 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.337+0000 7fc0c7d1e700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc0c00fb4a0 0x7fc0c00fc680 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:29.342 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.337+0000 7fc0c7d1e700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc0c0102e70 0x7fc0c00fcbc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:29.342 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.337+0000 7fc0c7d1e700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc0c00fd150 con 0x7fc0c0102e70 2024-04-02T04:55:29.342 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.337+0000 7fc0c7d1e700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc0c00fd290 con 0x7fc0c00fb4a0 2024-04-02T04:55:29.342 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.337+0000 7fc0c52b9700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc0c0102e70 0x7fc0c00fcbc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:29.343 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.337+0000 7fc0c52b9700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc0c0102e70 0x7fc0c00fcbc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:46306/0 (socket says 172.21.15.152:46306) 2024-04-02T04:55:29.343 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.337+0000 7fc0c52b9700 1 -- 172.21.15.152:0/3257567650 learned_addr learned my addr 172.21.15.152:0/3257567650 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:55:29.343 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.337+0000 7fc0c5aba700 1 --2- 172.21.15.152:0/3257567650 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc0c00fb4a0 0x7fc0c00fc680 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:29.343 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.337+0000 7fc0c52b9700 1 -- 172.21.15.152:0/3257567650 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc0c00fb4a0 msgr2=0x7fc0c00fc680 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:29.343 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.337+0000 7fc0c52b9700 1 --2- 172.21.15.152:0/3257567650 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc0c00fb4a0 0x7fc0c00fc680 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:29.343 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.337+0000 7fc0c52b9700 1 -- 172.21.15.152:0/3257567650 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc0b0005040 con 0x7fc0c0102e70 2024-04-02T04:55:29.343 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.337+0000 7fc0c5aba700 1 --2- 172.21.15.152:0/3257567650 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc0c00fb4a0 0x7fc0c00fc680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-02T04:55:29.343 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.337+0000 7fc0c52b9700 1 --2- 172.21.15.152:0/3257567650 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc0c0102e70 0x7fc0c00fcbc0 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7fc0b0009150 tx=0x7fc0b0009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:29.343 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.337+0000 7fc0b67fc700 1 -- 172.21.15.152:0/3257567650 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fc0b000e040 con 0x7fc0c0102e70 2024-04-02T04:55:29.343 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.337+0000 7fc0c7d1e700 1 -- 172.21.15.152:0/3257567650 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc0c00fd4e0 con 0x7fc0c0102e70 2024-04-02T04:55:29.343 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.337+0000 7fc0b67fc700 1 -- 172.21.15.152:0/3257567650 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fc0b0013420 con 0x7fc0c0102e70 2024-04-02T04:55:29.343 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.337+0000 7fc0b67fc700 1 -- 172.21.15.152:0/3257567650 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fc0b001c630 con 0x7fc0c0102e70 2024-04-02T04:55:29.343 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.337+0000 7fc0c7d1e700 1 -- 172.21.15.152:0/3257567650 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc0c011a2a0 con 0x7fc0c0102e70 2024-04-02T04:55:29.344 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.338+0000 7fc0c7d1e700 1 -- 172.21.15.152:0/3257567650 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc0c0100110 con 0x7fc0c0102e70 2024-04-02T04:55:29.344 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.339+0000 7fc0b67fc700 1 -- 172.21.15.152:0/3257567650 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7fc0b0013590 con 0x7fc0c0102e70 2024-04-02T04:55:29.345 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.340+0000 7fc0b67fc700 1 --2- 172.21.15.152:0/3257567650 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fc0ac0692b0 0x7fc0ac06b760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:29.345 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.340+0000 7fc0b67fc700 1 -- 172.21.15.152:0/3257567650 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7fc0b0088b00 con 0x7fc0c0102e70 2024-04-02T04:55:29.345 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.340+0000 7fc0c5aba700 1 --2- 172.21.15.152:0/3257567650 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fc0ac0692b0 0x7fc0ac06b760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:29.345 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.340+0000 7fc0c5aba700 1 --2- 172.21.15.152:0/3257567650 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fc0ac0692b0 0x7fc0ac06b760 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7fc0bc0041d0 tx=0x7fc0bc009040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:29.347 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.343+0000 7fc0b67fc700 1 -- 172.21.15.152:0/3257567650 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fc0b0055210 con 0x7fc0c0102e70 2024-04-02T04:55:29.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:29 smithi154 ceph-mon[35729]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:29.584 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.580+0000 7fc0c7d1e700 1 -- 172.21.15.152:0/3257567650 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "health", "format": "json"} v 0) v1 -- 0x7fc0c0065400 con 0x7fc0c0102e70 2024-04-02T04:55:29.584 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.580+0000 7fc0b67fc700 1 -- 172.21.15.152:0/3257567650 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "health", "format": "json"}]=0 v0) v1 ==== 72+0+46 (secure 0 0 0) 0x7fc0b005ce60 con 0x7fc0c0102e70 2024-04-02T04:55:29.585 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:55:29.585 INFO:teuthology.orchestra.run.smithi152.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-02T04:55:29.587 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.582+0000 7fc0c7d1e700 1 -- 172.21.15.152:0/3257567650 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fc0ac0692b0 msgr2=0x7fc0ac06b760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:29.587 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.582+0000 7fc0c7d1e700 1 --2- 172.21.15.152:0/3257567650 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fc0ac0692b0 0x7fc0ac06b760 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 crypto rx=0x7fc0bc0041d0 tx=0x7fc0bc009040 comp rx=0 tx=0).stop 2024-04-02T04:55:29.587 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.583+0000 7fc0c7d1e700 1 -- 172.21.15.152:0/3257567650 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc0c0102e70 msgr2=0x7fc0c00fcbc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:29.587 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.583+0000 7fc0c7d1e700 1 --2- 172.21.15.152:0/3257567650 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc0c0102e70 0x7fc0c00fcbc0 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 crypto rx=0x7fc0b0009150 tx=0x7fc0b0009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:29.588 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.583+0000 7fc0c7d1e700 1 -- 172.21.15.152:0/3257567650 shutdown_connections 2024-04-02T04:55:29.588 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.583+0000 7fc0c7d1e700 1 --2- 172.21.15.152:0/3257567650 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fc0ac0692b0 0x7fc0ac06b760 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:29.588 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.583+0000 7fc0c7d1e700 1 --2- 172.21.15.152:0/3257567650 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc0c00fb4a0 0x7fc0c00fc680 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:29.588 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.583+0000 7fc0c7d1e700 1 --2- 172.21.15.152:0/3257567650 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc0c0102e70 0x7fc0c00fcbc0 unknown :-1 s=CLOSED pgs=220 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:29.588 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.583+0000 7fc0c7d1e700 1 -- 172.21.15.152:0/3257567650 >> 172.21.15.152:0/3257567650 conn(0x7fc0c00714c0 msgr2=0x7fc0c00fa810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:29.588 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.583+0000 7fc0c7d1e700 1 -- 172.21.15.152:0/3257567650 shutdown_connections 2024-04-02T04:55:29.589 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:29.583+0000 7fc0c7d1e700 1 -- 172.21.15.152:0/3257567650 wait complete. 2024-04-02T04:55:30.234 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:30 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/3257567650' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-02T04:55:30.260 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-02T04:55:30.260 INFO:tasks.cephadm:Setup complete, yielding 2024-04-02T04:55:30.260 INFO:teuthology.run_tasks:Running task print... 2024-04-02T04:55:30.270 INFO:teuthology.task.print:**** done end installing quincy cephadm ... 2024-04-02T04:55:30.270 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-02T04:55:30.279 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi152.front.sepia.ceph.com 2024-04-02T04:55:30.279 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- bash -c 'ceph config set mgr mgr/cephadm/use_repo_digest true --force' 2024-04-02T04:55:30.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:30 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/3257567650' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-02T04:55:30.538 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:55:31.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:31 smithi152 ceph-mon[29073]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:31.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:31 smithi154 ceph-mon[35729]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:32.191 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.186+0000 7f432c00a700 1 -- 172.21.15.152:0/2116235649 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f43240fbcd0 msgr2=0x7f43240fc0e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:32.192 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.186+0000 7f432c00a700 1 --2- 172.21.15.152:0/2116235649 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f43240fbcd0 0x7f43240fc0e0 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 crypto rx=0x7f4320004940 tx=0x7f4320009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:32.192 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.186+0000 7f432c00a700 1 -- 172.21.15.152:0/2116235649 shutdown_connections 2024-04-02T04:55:32.192 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.186+0000 7f432c00a700 1 --2- 172.21.15.152:0/2116235649 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f43240fcf80 0x7f43240fd3f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:32.192 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.186+0000 7f432c00a700 1 --2- 172.21.15.152:0/2116235649 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f43240fbcd0 0x7f43240fc0e0 unknown :-1 s=CLOSED pgs=221 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:32.192 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.186+0000 7f432c00a700 1 -- 172.21.15.152:0/2116235649 >> 172.21.15.152:0/2116235649 conn(0x7f43240f7450 msgr2=0x7f43240f9860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:32.192 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.186+0000 7f432c00a700 1 -- 172.21.15.152:0/2116235649 shutdown_connections 2024-04-02T04:55:32.192 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.186+0000 7f432c00a700 1 -- 172.21.15.152:0/2116235649 wait complete. 2024-04-02T04:55:32.192 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.187+0000 7f432c00a700 1 Processor -- start 2024-04-02T04:55:32.192 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.187+0000 7f432c00a700 1 -- start start 2024-04-02T04:55:32.192 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.187+0000 7f432c00a700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f43240fbcd0 0x7f4324066cd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:32.192 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.187+0000 7f432c00a700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f43240fcf80 0x7f4324112720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:32.192 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.187+0000 7f432c00a700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4324112c60 con 0x7f43240fcf80 2024-04-02T04:55:32.192 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.187+0000 7f432c00a700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4324112da0 con 0x7f43240fbcd0 2024-04-02T04:55:32.193 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.187+0000 7f43295a5700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f43240fcf80 0x7f4324112720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:32.193 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.187+0000 7f43295a5700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f43240fcf80 0x7f4324112720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:55428/0 (socket says 172.21.15.152:55428) 2024-04-02T04:55:32.193 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.187+0000 7f43295a5700 1 -- 172.21.15.152:0/3966846885 learned_addr learned my addr 172.21.15.152:0/3966846885 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:55:32.193 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.187+0000 7f4329da6700 1 --2- 172.21.15.152:0/3966846885 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f43240fbcd0 0x7f4324066cd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:32.193 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.187+0000 7f43295a5700 1 -- 172.21.15.152:0/3966846885 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f43240fbcd0 msgr2=0x7f4324066cd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:32.193 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.187+0000 7f43295a5700 1 --2- 172.21.15.152:0/3966846885 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f43240fbcd0 0x7f4324066cd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:32.193 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.188+0000 7f43295a5700 1 -- 172.21.15.152:0/3966846885 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4320005040 con 0x7f43240fcf80 2024-04-02T04:55:32.193 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.188+0000 7f43295a5700 1 --2- 172.21.15.152:0/3966846885 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f43240fcf80 0x7f4324112720 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f4314002a50 tx=0x7f43140046b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:32.193 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.188+0000 7f431a7fc700 1 -- 172.21.15.152:0/3966846885 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f4314007cc0 con 0x7f43240fcf80 2024-04-02T04:55:32.193 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.188+0000 7f431a7fc700 1 -- 172.21.15.152:0/3966846885 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f431400f040 con 0x7f43240fcf80 2024-04-02T04:55:32.193 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.188+0000 7f432c00a700 1 -- 172.21.15.152:0/3966846885 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4324113000 con 0x7f43240fcf80 2024-04-02T04:55:32.193 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.188+0000 7f431a7fc700 1 -- 172.21.15.152:0/3966846885 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f4314009640 con 0x7f43240fcf80 2024-04-02T04:55:32.193 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.188+0000 7f432c00a700 1 -- 172.21.15.152:0/3966846885 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4324113590 con 0x7f43240fcf80 2024-04-02T04:55:32.194 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.189+0000 7f432c00a700 1 -- 172.21.15.152:0/3966846885 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f432404c710 con 0x7f43240fcf80 2024-04-02T04:55:32.194 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.189+0000 7f431a7fc700 1 -- 172.21.15.152:0/3966846885 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f43140097a0 con 0x7f43240fcf80 2024-04-02T04:55:32.194 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.190+0000 7f431a7fc700 1 --2- 172.21.15.152:0/3966846885 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f43100692a0 0x7f431006b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:32.194 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.190+0000 7f4329da6700 1 --2- 172.21.15.152:0/3966846885 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f43100692a0 0x7f431006b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:32.194 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.190+0000 7f431a7fc700 1 -- 172.21.15.152:0/3966846885 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7f4314086c60 con 0x7f43240fcf80 2024-04-02T04:55:32.195 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.190+0000 7f4329da6700 1 --2- 172.21.15.152:0/3966846885 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f43100692a0 0x7f431006b750 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f432000b3b0 tx=0x7f4320009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:32.197 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.194+0000 7f431a7fc700 1 -- 172.21.15.152:0/3966846885 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f4314056ce0 con 0x7f43240fcf80 2024-04-02T04:55:32.385 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.381+0000 7f432c00a700 1 -- 172.21.15.152:0/3966846885 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command([{prefix=config set, name=mgr/cephadm/use_repo_digest}] v 0) v1 -- 0x7f43241137e0 con 0x7f43240fcf80 2024-04-02T04:55:32.391 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.387+0000 7f431a7fc700 1 -- 172.21.15.152:0/3966846885 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/cephadm/use_repo_digest}]=0 v20)=0 v20) v1 ==== 143+0+0 (secure 0 0 0) 0x7f4314056870 con 0x7f43240fcf80 2024-04-02T04:55:32.393 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.389+0000 7f432c00a700 1 -- 172.21.15.152:0/3966846885 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f43100692a0 msgr2=0x7f431006b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:32.393 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.389+0000 7f432c00a700 1 --2- 172.21.15.152:0/3966846885 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f43100692a0 0x7f431006b750 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 crypto rx=0x7f432000b3b0 tx=0x7f4320009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:32.393 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.389+0000 7f432c00a700 1 -- 172.21.15.152:0/3966846885 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f43240fcf80 msgr2=0x7f4324112720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:32.394 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.389+0000 7f432c00a700 1 --2- 172.21.15.152:0/3966846885 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f43240fcf80 0x7f4324112720 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 crypto rx=0x7f4314002a50 tx=0x7f43140046b0 comp rx=0 tx=0).stop 2024-04-02T04:55:32.394 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.390+0000 7f432c00a700 1 -- 172.21.15.152:0/3966846885 shutdown_connections 2024-04-02T04:55:32.394 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.390+0000 7f432c00a700 1 --2- 172.21.15.152:0/3966846885 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f43100692a0 0x7f431006b750 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:32.394 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.390+0000 7f432c00a700 1 --2- 172.21.15.152:0/3966846885 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f43240fbcd0 0x7f4324066cd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:32.394 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.390+0000 7f432c00a700 1 --2- 172.21.15.152:0/3966846885 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f43240fcf80 0x7f4324112720 unknown :-1 s=CLOSED pgs=222 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:32.394 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.390+0000 7f432c00a700 1 -- 172.21.15.152:0/3966846885 >> 172.21.15.152:0/3966846885 conn(0x7f43240f7450 msgr2=0x7f43240f8b30 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:32.394 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.390+0000 7f432c00a700 1 -- 172.21.15.152:0/3966846885 shutdown_connections 2024-04-02T04:55:32.394 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:32.390+0000 7f432c00a700 1 -- 172.21.15.152:0/3966846885 wait complete. 2024-04-02T04:55:33.059 INFO:teuthology.run_tasks:Running task print... 2024-04-02T04:55:33.068 INFO:teuthology.task.print:**** done cephadm.shell ceph config set mgr... 2024-04-02T04:55:33.068 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-02T04:55:33.077 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi152.front.sepia.ceph.com 2024-04-02T04:55:33.077 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- bash -c 'ceph orch status' 2024-04-02T04:55:33.344 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:55:33.679 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:33 smithi152 ceph-mon[29073]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:33.679 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:33 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/3966846885' entity='client.admin' 2024-04-02T04:55:33.679 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:33 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:55:33.679 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:33 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:55:33.679 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:33 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:55:33.679 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:33 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:55:33.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:33 smithi154 ceph-mon[35729]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:33.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:33 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/3966846885' entity='client.admin' 2024-04-02T04:55:33.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:33 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:55:33.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:33 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:55:33.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:33 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:55:33.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:33 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:55:34.938 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.932+0000 7f9f44a87700 1 -- 172.21.15.152:0/3263521951 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f9f400fb1f0 msgr2=0x7f9f400fb600 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:34.938 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.932+0000 7f9f44a87700 1 --2- 172.21.15.152:0/3263521951 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f9f400fb1f0 0x7f9f400fb600 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 crypto rx=0x7f9f30004940 tx=0x7f9f30009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:34.938 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.932+0000 7f9f44a87700 1 -- 172.21.15.152:0/3263521951 shutdown_connections 2024-04-02T04:55:34.938 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.932+0000 7f9f44a87700 1 --2- 172.21.15.152:0/3263521951 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9f400fbb40 0x7f9f400fe3a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:34.938 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.932+0000 7f9f44a87700 1 --2- 172.21.15.152:0/3263521951 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f9f400fb1f0 0x7f9f400fb600 unknown :-1 s=CLOSED pgs=223 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:34.938 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.932+0000 7f9f44a87700 1 -- 172.21.15.152:0/3263521951 >> 172.21.15.152:0/3263521951 conn(0x7f9f400f6bd0 msgr2=0x7f9f400f9000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:34.938 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.932+0000 7f9f44a87700 1 -- 172.21.15.152:0/3263521951 shutdown_connections 2024-04-02T04:55:34.938 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.932+0000 7f9f44a87700 1 -- 172.21.15.152:0/3263521951 wait complete. 2024-04-02T04:55:34.939 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.932+0000 7f9f44a87700 1 Processor -- start 2024-04-02T04:55:34.939 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.932+0000 7f9f44a87700 1 -- start start 2024-04-02T04:55:34.939 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.932+0000 7f9f44a87700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9f400fb1f0 0x7f9f4010db40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:34.939 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.932+0000 7f9f44a87700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f9f400fbb40 0x7f9f4010e080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:34.939 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.932+0000 7f9f44a87700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f4010e680 con 0x7f9f400fbb40 2024-04-02T04:55:34.939 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.932+0000 7f9f44a87700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f4010e7f0 con 0x7f9f400fb1f0 2024-04-02T04:55:34.939 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.933+0000 7f9f3dd9b700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f9f400fbb40 0x7f9f4010e080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:34.939 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.933+0000 7f9f3e59c700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9f400fb1f0 0x7f9f4010db40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:34.939 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.933+0000 7f9f3e59c700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9f400fb1f0 0x7f9f4010db40 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.152:55334/0 (socket says 172.21.15.152:55334) 2024-04-02T04:55:34.939 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.933+0000 7f9f3dd9b700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f9f400fbb40 0x7f9f4010e080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:55444/0 (socket says 172.21.15.152:55444) 2024-04-02T04:55:34.939 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.933+0000 7f9f3e59c700 1 -- 172.21.15.152:0/4218216011 learned_addr learned my addr 172.21.15.152:0/4218216011 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:55:34.939 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.933+0000 7f9f3e59c700 1 -- 172.21.15.152:0/4218216011 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f9f400fbb40 msgr2=0x7f9f4010e080 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:34.939 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.933+0000 7f9f3e59c700 1 --2- 172.21.15.152:0/4218216011 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f9f400fbb40 0x7f9f4010e080 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:34.940 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.933+0000 7f9f3e59c700 1 -- 172.21.15.152:0/4218216011 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9f30005040 con 0x7f9f400fb1f0 2024-04-02T04:55:34.940 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.933+0000 7f9f3dd9b700 1 --2- 172.21.15.152:0/4218216011 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f9f400fbb40 0x7f9f4010e080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-02T04:55:34.940 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.933+0000 7f9f3e59c700 1 --2- 172.21.15.152:0/4218216011 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9f400fb1f0 0x7f9f4010db40 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7f9f3000b3b0 tx=0x7f9f30009e30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:34.940 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.933+0000 7f9f2effd700 1 -- 172.21.15.152:0/4218216011 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f9f3000e040 con 0x7f9f400fb1f0 2024-04-02T04:55:34.940 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.933+0000 7f9f2effd700 1 -- 172.21.15.152:0/4218216011 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f9f3000bbf0 con 0x7f9f400fb1f0 2024-04-02T04:55:34.940 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.933+0000 7f9f44a87700 1 -- 172.21.15.152:0/4218216011 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9f401131f0 con 0x7f9f400fb1f0 2024-04-02T04:55:34.940 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.933+0000 7f9f2effd700 1 -- 172.21.15.152:0/4218216011 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f9f3001c630 con 0x7f9f400fb1f0 2024-04-02T04:55:34.940 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.933+0000 7f9f44a87700 1 -- 172.21.15.152:0/4218216011 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9f401137b0 con 0x7f9f400fb1f0 2024-04-02T04:55:34.940 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.934+0000 7f9f44a87700 1 -- 172.21.15.152:0/4218216011 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9f4004c710 con 0x7f9f400fb1f0 2024-04-02T04:55:34.941 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.935+0000 7f9f2effd700 1 -- 172.21.15.152:0/4218216011 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f9f3000bd60 con 0x7f9f400fb1f0 2024-04-02T04:55:34.941 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.935+0000 7f9f2effd700 1 --2- 172.21.15.152:0/4218216011 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f9f280692a0 0x7f9f2806b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:34.941 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.935+0000 7f9f2effd700 1 -- 172.21.15.152:0/4218216011 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7f9f30087c50 con 0x7f9f400fb1f0 2024-04-02T04:55:34.941 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.935+0000 7f9f3dd9b700 1 --2- 172.21.15.152:0/4218216011 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f9f280692a0 0x7f9f2806b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:34.941 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.936+0000 7f9f3dd9b700 1 --2- 172.21.15.152:0/4218216011 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f9f280692a0 0x7f9f2806b750 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f9f34010520 tx=0x7f9f3400b040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:34.943 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:34.939+0000 7f9f2effd700 1 -- 172.21.15.152:0/4218216011 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f9f30057df0 con 0x7f9f400fb1f0 2024-04-02T04:55:35.144 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:35.140+0000 7f9f44a87700 1 -- 172.21.15.152:0/4218216011 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "orch status", "target": ["mon-mgr", ""]}) v1 -- 0x7f9f401026e0 con 0x7f9f280692a0 2024-04-02T04:55:35.145 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:35.141+0000 7f9f2effd700 1 -- 172.21.15.152:0/4218216011 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+43 (secure 0 0 0) 0x7f9f401026e0 con 0x7f9f280692a0 2024-04-02T04:55:35.145 INFO:teuthology.orchestra.run.smithi152.stdout:Backend: cephadm 2024-04-02T04:55:35.145 INFO:teuthology.orchestra.run.smithi152.stdout:Available: Yes 2024-04-02T04:55:35.145 INFO:teuthology.orchestra.run.smithi152.stdout:Paused: No 2024-04-02T04:55:35.147 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:35.143+0000 7f9f44a87700 1 -- 172.21.15.152:0/4218216011 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f9f280692a0 msgr2=0x7f9f2806b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:35.147 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:35.143+0000 7f9f44a87700 1 --2- 172.21.15.152:0/4218216011 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f9f280692a0 0x7f9f2806b750 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 crypto rx=0x7f9f34010520 tx=0x7f9f3400b040 comp rx=0 tx=0).stop 2024-04-02T04:55:35.147 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:35.143+0000 7f9f44a87700 1 -- 172.21.15.152:0/4218216011 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9f400fb1f0 msgr2=0x7f9f4010db40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:35.147 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:35.143+0000 7f9f44a87700 1 --2- 172.21.15.152:0/4218216011 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9f400fb1f0 0x7f9f4010db40 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 crypto rx=0x7f9f3000b3b0 tx=0x7f9f30009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:35.147 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:35.143+0000 7f9f44a87700 1 -- 172.21.15.152:0/4218216011 shutdown_connections 2024-04-02T04:55:35.147 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:35.143+0000 7f9f44a87700 1 --2- 172.21.15.152:0/4218216011 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f9f280692a0 0x7f9f2806b750 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:35.147 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:35.143+0000 7f9f44a87700 1 --2- 172.21.15.152:0/4218216011 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9f400fb1f0 0x7f9f4010db40 unknown :-1 s=CLOSED pgs=34 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:35.148 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:35.143+0000 7f9f44a87700 1 --2- 172.21.15.152:0/4218216011 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f9f400fbb40 0x7f9f4010e080 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:35.148 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:35.143+0000 7f9f44a87700 1 -- 172.21.15.152:0/4218216011 >> 172.21.15.152:0/4218216011 conn(0x7f9f400f6bd0 msgr2=0x7f9f400f8290 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:35.148 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:35.144+0000 7f9f44a87700 1 -- 172.21.15.152:0/4218216011 shutdown_connections 2024-04-02T04:55:35.148 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:35.144+0000 7f9f44a87700 1 -- 172.21.15.152:0/4218216011 wait complete. 2024-04-02T04:55:35.482 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:35 smithi152 ceph-mon[29073]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:35.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:35 smithi154 ceph-mon[35729]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:35.826 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- bash -c 'ceph orch ps' 2024-04-02T04:55:36.080 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:55:36.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:36 smithi152 ceph-mon[29073]: from='client.24263 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:55:36.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:36 smithi154 ceph-mon[35729]: from='client.24263 -' entity='client.admin' cmd=[{"prefix": "orch status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:55:37.611 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.607+0000 7f2e3e234700 1 -- 172.21.15.152:0/3027502359 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2e380fee60 msgr2=0x7f2e380fafa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:37.611 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.607+0000 7f2e3e234700 1 --2- 172.21.15.152:0/3027502359 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2e380fee60 0x7f2e380fafa0 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 crypto rx=0x7f2e20004940 tx=0x7f2e20009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:37.612 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.607+0000 7f2e3e234700 1 -- 172.21.15.152:0/3027502359 shutdown_connections 2024-04-02T04:55:37.612 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.607+0000 7f2e3e234700 1 --2- 172.21.15.152:0/3027502359 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2e380fee60 0x7f2e380fafa0 unknown :-1 s=CLOSED pgs=224 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:37.612 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.607+0000 7f2e3e234700 1 --2- 172.21.15.152:0/3027502359 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2e380fe510 0x7f2e380fe920 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:37.612 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.607+0000 7f2e3e234700 1 -- 172.21.15.152:0/3027502359 >> 172.21.15.152:0/3027502359 conn(0x7f2e380f6c10 msgr2=0x7f2e380f9020 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:37.612 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.607+0000 7f2e3e234700 1 -- 172.21.15.152:0/3027502359 shutdown_connections 2024-04-02T04:55:37.612 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.608+0000 7f2e3e234700 1 -- 172.21.15.152:0/3027502359 wait complete. 2024-04-02T04:55:37.613 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.608+0000 7f2e3e234700 1 Processor -- start 2024-04-02T04:55:37.613 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.608+0000 7f2e3e234700 1 -- start start 2024-04-02T04:55:37.613 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.609+0000 7f2e3e234700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2e380fe510 0x7f2e3810d940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:37.614 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.609+0000 7f2e3e234700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2e380fee60 0x7f2e3810de80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:37.614 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.609+0000 7f2e3e234700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2e3810e480 con 0x7f2e380fee60 2024-04-02T04:55:37.614 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.609+0000 7f2e3e234700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2e3810e5f0 con 0x7f2e380fe510 2024-04-02T04:55:37.614 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.609+0000 7f2e36ffd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2e380fee60 0x7f2e3810de80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:37.614 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.609+0000 7f2e36ffd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2e380fee60 0x7f2e3810de80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:55474/0 (socket says 172.21.15.152:55474) 2024-04-02T04:55:37.614 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.609+0000 7f2e36ffd700 1 -- 172.21.15.152:0/181115170 learned_addr learned my addr 172.21.15.152:0/181115170 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:55:37.614 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.609+0000 7f2e377fe700 1 --2- 172.21.15.152:0/181115170 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2e380fe510 0x7f2e3810d940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:37.614 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.609+0000 7f2e36ffd700 1 -- 172.21.15.152:0/181115170 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2e380fe510 msgr2=0x7f2e3810d940 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:37.614 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.609+0000 7f2e36ffd700 1 --2- 172.21.15.152:0/181115170 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2e380fe510 0x7f2e3810d940 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:37.614 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.609+0000 7f2e36ffd700 1 -- 172.21.15.152:0/181115170 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2e20005040 con 0x7f2e380fee60 2024-04-02T04:55:37.614 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.609+0000 7f2e36ffd700 1 --2- 172.21.15.152:0/181115170 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2e380fee60 0x7f2e3810de80 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7f2e2000b3b0 tx=0x7f2e20009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:37.614 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.609+0000 7f2e2ffff700 1 -- 172.21.15.152:0/181115170 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f2e2000e040 con 0x7f2e380fee60 2024-04-02T04:55:37.614 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.609+0000 7f2e2ffff700 1 -- 172.21.15.152:0/181115170 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f2e2000bbf0 con 0x7f2e380fee60 2024-04-02T04:55:37.614 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.609+0000 7f2e3e234700 1 -- 172.21.15.152:0/181115170 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2e38112ff0 con 0x7f2e380fee60 2024-04-02T04:55:37.615 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.609+0000 7f2e2ffff700 1 -- 172.21.15.152:0/181115170 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f2e2001c630 con 0x7f2e380fee60 2024-04-02T04:55:37.615 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.610+0000 7f2e3e234700 1 -- 172.21.15.152:0/181115170 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2e38113580 con 0x7f2e380fee60 2024-04-02T04:55:37.615 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.610+0000 7f2e3e234700 1 -- 172.21.15.152:0/181115170 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2e3804c710 con 0x7f2e380fee60 2024-04-02T04:55:37.615 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.611+0000 7f2e2ffff700 1 -- 172.21.15.152:0/181115170 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f2e2000bd60 con 0x7f2e380fee60 2024-04-02T04:55:37.616 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.611+0000 7f2e2ffff700 1 --2- 172.21.15.152:0/181115170 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2e240692a0 0x7f2e2406b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:37.616 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.611+0000 7f2e2ffff700 1 -- 172.21.15.152:0/181115170 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7f2e20087b30 con 0x7f2e380fee60 2024-04-02T04:55:37.616 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.611+0000 7f2e377fe700 1 --2- 172.21.15.152:0/181115170 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2e240692a0 0x7f2e2406b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:37.616 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.612+0000 7f2e377fe700 1 --2- 172.21.15.152:0/181115170 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2e240692a0 0x7f2e2406b750 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f2e28004f40 tx=0x7f2e2800b040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:37.619 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.616+0000 7f2e2ffff700 1 -- 172.21.15.152:0/181115170 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f2e20057c30 con 0x7f2e380fee60 2024-04-02T04:55:37.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:37 smithi154 ceph-mon[35729]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:37.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:37 smithi152 ceph-mon[29073]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:37.811 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.808+0000 7f2e3e234700 1 -- 172.21.15.152:0/181115170 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "orch ps", "target": ["mon-mgr", ""]}) v1 -- 0x7f2e381025e0 con 0x7f2e240692a0 2024-04-02T04:55:37.819 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.816+0000 7f2e2ffff700 1 -- 172.21.15.152:0/181115170 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+2682 (secure 0 0 0) 0x7f2e381025e0 con 0x7f2e240692a0 2024-04-02T04:55:37.820 INFO:teuthology.orchestra.run.smithi152.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T04:55:37.820 INFO:teuthology.orchestra.run.smithi152.stdout:alertmanager.smithi152 smithi152 *:9093,9094 running (3m) 2m ago 5m 26.3M - 0.25.0 c8568f914cd2 1f52e384342b 2024-04-02T04:55:37.820 INFO:teuthology.orchestra.run.smithi152.stdout:crash.smithi152 smithi152 running (5m) 2m ago 5m 6665k - 17.2.7-803-gc924f991 4948510fac0f 12f8a8489975 2024-04-02T04:55:37.820 INFO:teuthology.orchestra.run.smithi152.stdout:crash.smithi154 smithi154 running (4m) 67s ago 4m 6665k - 17.2.7-803-gc924f991 4948510fac0f 7fe215d49e0f 2024-04-02T04:55:37.820 INFO:teuthology.orchestra.run.smithi152.stdout:grafana.smithi152 smithi152 *:3000 running (3m) 2m ago 4m 78.0M - 9.4.7 954c08fa6188 eb6036b14f99 2024-04-02T04:55:37.820 INFO:teuthology.orchestra.run.smithi152.stdout:mgr.smithi152.jivdcl smithi152 *:9283 running (6m) 2m ago 6m 470M - 17.2.7-803-gc924f991 4948510fac0f 2fd5fbea3f34 2024-04-02T04:55:37.820 INFO:teuthology.orchestra.run.smithi152.stdout:mgr.smithi154.pplnxz smithi154 *:8443,9283 running (4m) 67s ago 4m 431M - 17.2.7-803-gc924f991 4948510fac0f 0a1df688b4ac 2024-04-02T04:55:37.820 INFO:teuthology.orchestra.run.smithi152.stdout:mon.smithi152 smithi152 running (6m) 2m ago 6m 40.5M 2048M 17.2.7-803-gc924f991 4948510fac0f 399b9cc778f0 2024-04-02T04:55:37.820 INFO:teuthology.orchestra.run.smithi152.stdout:mon.smithi154 smithi154 running (4m) 67s ago 4m 37.9M 2048M 17.2.7-803-gc924f991 4948510fac0f 975aefa2215f 2024-04-02T04:55:37.820 INFO:teuthology.orchestra.run.smithi152.stdout:node-exporter.smithi152 smithi152 *:9100 running (5m) 2m ago 5m 16.1M - 1.5.0 0da6a335fe13 8168d9a53e30 2024-04-02T04:55:37.820 INFO:teuthology.orchestra.run.smithi152.stdout:node-exporter.smithi154 smithi154 *:9100 running (4m) 67s ago 4m 15.3M - 1.5.0 0da6a335fe13 8ee2f7610052 2024-04-02T04:55:37.821 INFO:teuthology.orchestra.run.smithi152.stdout:osd.0 smithi152 running (3m) 2m ago 3m 34.9M 4288M 17.2.7-803-gc924f991 4948510fac0f 9935e01f3199 2024-04-02T04:55:37.821 INFO:teuthology.orchestra.run.smithi152.stdout:osd.1 smithi152 running (2m) 2m ago 2m 35.8M 4288M 17.2.7-803-gc924f991 4948510fac0f e5c49d238944 2024-04-02T04:55:37.821 INFO:teuthology.orchestra.run.smithi152.stdout:osd.2 smithi152 running (2m) 2m ago 2m 13.3M 4288M 17.2.7-803-gc924f991 4948510fac0f c53a93ae7737 2024-04-02T04:55:37.821 INFO:teuthology.orchestra.run.smithi152.stdout:osd.3 smithi154 running (2m) 67s ago 2m 40.4M 5312M 17.2.7-803-gc924f991 4948510fac0f 959ebaab3713 2024-04-02T04:55:37.821 INFO:teuthology.orchestra.run.smithi152.stdout:osd.4 smithi154 running (97s) 67s ago 96s 36.6M 5312M 17.2.7-803-gc924f991 4948510fac0f 4856c813e7fd 2024-04-02T04:55:37.821 INFO:teuthology.orchestra.run.smithi152.stdout:osd.5 smithi154 running (73s) 67s ago 72s 11.0M 5312M 17.2.7-803-gc924f991 4948510fac0f fbc6bb91d1a7 2024-04-02T04:55:37.821 INFO:teuthology.orchestra.run.smithi152.stdout:prometheus.smithi152 smithi152 *:9095 running (3m) 2m ago 4m 35.7M - 2.43.0 a07b618ecd1d 874955f0a102 2024-04-02T04:55:37.822 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.818+0000 7f2e3e234700 1 -- 172.21.15.152:0/181115170 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2e240692a0 msgr2=0x7f2e2406b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:37.822 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.818+0000 7f2e3e234700 1 --2- 172.21.15.152:0/181115170 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2e240692a0 0x7f2e2406b750 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 crypto rx=0x7f2e28004f40 tx=0x7f2e2800b040 comp rx=0 tx=0).stop 2024-04-02T04:55:37.822 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.818+0000 7f2e3e234700 1 -- 172.21.15.152:0/181115170 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2e380fee60 msgr2=0x7f2e3810de80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:37.822 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.818+0000 7f2e3e234700 1 --2- 172.21.15.152:0/181115170 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2e380fee60 0x7f2e3810de80 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 crypto rx=0x7f2e2000b3b0 tx=0x7f2e20009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:37.822 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.818+0000 7f2e3e234700 1 -- 172.21.15.152:0/181115170 shutdown_connections 2024-04-02T04:55:37.822 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.818+0000 7f2e3e234700 1 --2- 172.21.15.152:0/181115170 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2e240692a0 0x7f2e2406b750 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:37.823 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.818+0000 7f2e3e234700 1 --2- 172.21.15.152:0/181115170 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2e380fe510 0x7f2e3810d940 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:37.823 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.818+0000 7f2e3e234700 1 --2- 172.21.15.152:0/181115170 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2e380fee60 0x7f2e3810de80 unknown :-1 s=CLOSED pgs=225 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:37.823 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.818+0000 7f2e3e234700 1 -- 172.21.15.152:0/181115170 >> 172.21.15.152:0/181115170 conn(0x7f2e380f6c10 msgr2=0x7f2e380f82c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:37.823 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.819+0000 7f2e3e234700 1 -- 172.21.15.152:0/181115170 shutdown_connections 2024-04-02T04:55:37.823 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:37.819+0000 7f2e3e234700 1 -- 172.21.15.152:0/181115170 wait complete. 2024-04-02T04:55:38.432 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- bash -c 'ceph orch ls' 2024-04-02T04:55:38.684 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:55:39.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:39 smithi154 ceph-mon[35729]: from='client.14436 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:55:39.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:39 smithi154 ceph-mon[35729]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:39.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:39 smithi152 ceph-mon[29073]: from='client.14436 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:55:39.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:39 smithi152 ceph-mon[29073]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:40.368 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.362+0000 7ff7f85f7700 1 -- 172.21.15.152:0/1993099789 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff7f00feda0 msgr2=0x7ff7f00faee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:40.368 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.362+0000 7ff7f85f7700 1 --2- 172.21.15.152:0/1993099789 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff7f00feda0 0x7ff7f00faee0 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 crypto rx=0x7ff7e0004940 tx=0x7ff7e0009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:40.368 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.363+0000 7ff7f85f7700 1 -- 172.21.15.152:0/1993099789 shutdown_connections 2024-04-02T04:55:40.368 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.363+0000 7ff7f85f7700 1 --2- 172.21.15.152:0/1993099789 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff7f00feda0 0x7ff7f00faee0 unknown :-1 s=CLOSED pgs=226 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:40.368 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.363+0000 7ff7f85f7700 1 --2- 172.21.15.152:0/1993099789 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff7f00fe450 0x7ff7f00fe860 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:40.368 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.363+0000 7ff7f85f7700 1 -- 172.21.15.152:0/1993099789 >> 172.21.15.152:0/1993099789 conn(0x7ff7f00f6c10 msgr2=0x7ff7f00f9020 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:40.368 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.363+0000 7ff7f85f7700 1 -- 172.21.15.152:0/1993099789 shutdown_connections 2024-04-02T04:55:40.368 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.363+0000 7ff7f85f7700 1 -- 172.21.15.152:0/1993099789 wait complete. 2024-04-02T04:55:40.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.363+0000 7ff7f85f7700 1 Processor -- start 2024-04-02T04:55:40.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.363+0000 7ff7f85f7700 1 -- start start 2024-04-02T04:55:40.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.363+0000 7ff7f85f7700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff7f00fe450 0x7ff7f010fb10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:40.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.363+0000 7ff7f85f7700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff7f00feda0 0x7ff7f0110050 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:40.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.363+0000 7ff7f85f7700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff7f01106e0 con 0x7ff7f00fe450 2024-04-02T04:55:40.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.363+0000 7ff7f85f7700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff7f0115080 con 0x7ff7f00feda0 2024-04-02T04:55:40.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.363+0000 7ff7f6393700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff7f00fe450 0x7ff7f010fb10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:40.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.363+0000 7ff7f6393700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff7f00fe450 0x7ff7f010fb10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:55486/0 (socket says 172.21.15.152:55486) 2024-04-02T04:55:40.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.363+0000 7ff7f6393700 1 -- 172.21.15.152:0/2895765815 learned_addr learned my addr 172.21.15.152:0/2895765815 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:55:40.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.364+0000 7ff7f5b92700 1 --2- 172.21.15.152:0/2895765815 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff7f00feda0 0x7ff7f0110050 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:40.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.364+0000 7ff7f6393700 1 -- 172.21.15.152:0/2895765815 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff7f00feda0 msgr2=0x7ff7f0110050 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:40.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.364+0000 7ff7f6393700 1 --2- 172.21.15.152:0/2895765815 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff7f00feda0 0x7ff7f0110050 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:40.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.364+0000 7ff7f6393700 1 -- 172.21.15.152:0/2895765815 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff7e0005040 con 0x7ff7f00fe450 2024-04-02T04:55:40.369 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.364+0000 7ff7f6393700 1 --2- 172.21.15.152:0/2895765815 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff7f00fe450 0x7ff7f010fb10 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7ff7ec00b790 tx=0x7ff7ec009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:40.370 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.364+0000 7ff7e6ffd700 1 -- 172.21.15.152:0/2895765815 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7ff7ec00f6f0 con 0x7ff7f00fe450 2024-04-02T04:55:40.370 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.364+0000 7ff7e6ffd700 1 -- 172.21.15.152:0/2895765815 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7ff7ec00fd20 con 0x7ff7f00fe450 2024-04-02T04:55:40.370 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.364+0000 7ff7f85f7700 1 -- 172.21.15.152:0/2895765815 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff7f01152e0 con 0x7ff7f00fe450 2024-04-02T04:55:40.370 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.364+0000 7ff7e6ffd700 1 -- 172.21.15.152:0/2895765815 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7ff7ec0103a0 con 0x7ff7f00fe450 2024-04-02T04:55:40.370 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.364+0000 7ff7f85f7700 1 -- 172.21.15.152:0/2895765815 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff7f0115850 con 0x7ff7f00fe450 2024-04-02T04:55:40.371 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.365+0000 7ff7f85f7700 1 -- 172.21.15.152:0/2895765815 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff7f0101b90 con 0x7ff7f00fe450 2024-04-02T04:55:40.371 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.366+0000 7ff7e6ffd700 1 -- 172.21.15.152:0/2895765815 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7ff7ec011400 con 0x7ff7f00fe450 2024-04-02T04:55:40.371 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.366+0000 7ff7e6ffd700 1 --2- 172.21.15.152:0/2895765815 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7ff7dc0692a0 0x7ff7dc06b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:40.371 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.366+0000 7ff7e6ffd700 1 -- 172.21.15.152:0/2895765815 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7ff7ec086c30 con 0x7ff7f00fe450 2024-04-02T04:55:40.371 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.366+0000 7ff7f5b92700 1 --2- 172.21.15.152:0/2895765815 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7ff7dc0692a0 0x7ff7dc06b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:40.371 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.366+0000 7ff7f5b92700 1 --2- 172.21.15.152:0/2895765815 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7ff7dc0692a0 0x7ff7dc06b750 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7ff7e000b970 tx=0x7ff7e0009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:40.374 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.370+0000 7ff7e6ffd700 1 -- 172.21.15.152:0/2895765815 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7ff7ec056d30 con 0x7ff7f00fe450 2024-04-02T04:55:40.573 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.570+0000 7ff7f85f7700 1 -- 172.21.15.152:0/2895765815 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "orch ls", "target": ["mon-mgr", ""]}) v1 -- 0x7ff7f002b340 con 0x7ff7dc0692a0 2024-04-02T04:55:40.577 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.574+0000 7ff7e6ffd700 1 -- 172.21.15.152:0/2895765815 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+1179 (secure 0 0 0) 0x7ff7f002b340 con 0x7ff7dc0692a0 2024-04-02T04:55:40.578 INFO:teuthology.orchestra.run.smithi152.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-02T04:55:40.578 INFO:teuthology.orchestra.run.smithi152.stdout:alertmanager ?:9093,9094 1/1 2m ago 5m count:1 2024-04-02T04:55:40.578 INFO:teuthology.orchestra.run.smithi152.stdout:crash 2/2 2m ago 5m * 2024-04-02T04:55:40.578 INFO:teuthology.orchestra.run.smithi152.stdout:grafana ?:3000 1/1 2m ago 5m count:1 2024-04-02T04:55:40.578 INFO:teuthology.orchestra.run.smithi152.stdout:mgr 2/2 2m ago 5m count:2 2024-04-02T04:55:40.578 INFO:teuthology.orchestra.run.smithi152.stdout:mon 2/2 2m ago 5m smithi152:172.21.15.152=smithi152;smithi154:172.21.15.154=smithi154;count:2 2024-04-02T04:55:40.578 INFO:teuthology.orchestra.run.smithi152.stdout:node-exporter ?:9100 2/2 2m ago 5m * 2024-04-02T04:55:40.578 INFO:teuthology.orchestra.run.smithi152.stdout:osd 6 2m ago - 2024-04-02T04:55:40.578 INFO:teuthology.orchestra.run.smithi152.stdout:prometheus ?:9095 1/1 2m ago 5m count:1 2024-04-02T04:55:40.580 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.576+0000 7ff7f85f7700 1 -- 172.21.15.152:0/2895765815 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7ff7dc0692a0 msgr2=0x7ff7dc06b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:40.580 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.576+0000 7ff7f85f7700 1 --2- 172.21.15.152:0/2895765815 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7ff7dc0692a0 0x7ff7dc06b750 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 crypto rx=0x7ff7e000b970 tx=0x7ff7e0009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:40.580 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.576+0000 7ff7f85f7700 1 -- 172.21.15.152:0/2895765815 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff7f00fe450 msgr2=0x7ff7f010fb10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:40.580 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.576+0000 7ff7f85f7700 1 --2- 172.21.15.152:0/2895765815 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff7f00fe450 0x7ff7f010fb10 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 crypto rx=0x7ff7ec00b790 tx=0x7ff7ec009250 comp rx=0 tx=0).stop 2024-04-02T04:55:40.580 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.576+0000 7ff7f85f7700 1 -- 172.21.15.152:0/2895765815 shutdown_connections 2024-04-02T04:55:40.580 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.576+0000 7ff7f85f7700 1 --2- 172.21.15.152:0/2895765815 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7ff7dc0692a0 0x7ff7dc06b750 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:40.580 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.576+0000 7ff7f85f7700 1 --2- 172.21.15.152:0/2895765815 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff7f00fe450 0x7ff7f010fb10 unknown :-1 s=CLOSED pgs=227 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:40.580 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.576+0000 7ff7f85f7700 1 --2- 172.21.15.152:0/2895765815 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff7f00feda0 0x7ff7f0110050 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:40.581 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.576+0000 7ff7f85f7700 1 -- 172.21.15.152:0/2895765815 >> 172.21.15.152:0/2895765815 conn(0x7ff7f00f6c10 msgr2=0x7ff7f00f8290 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:40.581 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.576+0000 7ff7f85f7700 1 -- 172.21.15.152:0/2895765815 shutdown_connections 2024-04-02T04:55:40.581 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:40.576+0000 7ff7f85f7700 1 -- 172.21.15.152:0/2895765815 wait complete. 2024-04-02T04:55:40.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:40 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:55:40.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:40 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:55:41.104 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- bash -c 'ceph orch host ls' 2024-04-02T04:55:41.360 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:55:41.717 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:41 smithi152 ceph-mon[29073]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:41.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:41 smithi154 ceph-mon[35729]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:42.592 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:42 smithi152 ceph-mon[29073]: from='client.14440 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:55:42.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:42 smithi154 ceph-mon[35729]: from='client.14440 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:55:42.960 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.954+0000 7fbaa4e29700 1 -- 172.21.15.152:0/1681359075 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbaa00fbcd0 msgr2=0x7fbaa00fc0e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:42.960 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.954+0000 7fbaa4e29700 1 --2- 172.21.15.152:0/1681359075 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbaa00fbcd0 0x7fbaa00fc0e0 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 crypto rx=0x7fba94004940 tx=0x7fba94009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:42.960 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.955+0000 7fbaa4e29700 1 -- 172.21.15.152:0/1681359075 shutdown_connections 2024-04-02T04:55:42.960 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.955+0000 7fbaa4e29700 1 --2- 172.21.15.152:0/1681359075 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbaa00fcf80 0x7fbaa00fd3f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:42.960 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.955+0000 7fbaa4e29700 1 --2- 172.21.15.152:0/1681359075 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbaa00fbcd0 0x7fbaa00fc0e0 unknown :-1 s=CLOSED pgs=228 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:42.960 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.955+0000 7fbaa4e29700 1 -- 172.21.15.152:0/1681359075 >> 172.21.15.152:0/1681359075 conn(0x7fbaa00f7450 msgr2=0x7fbaa00f9860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:42.961 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.955+0000 7fbaa4e29700 1 -- 172.21.15.152:0/1681359075 shutdown_connections 2024-04-02T04:55:42.961 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.955+0000 7fbaa4e29700 1 -- 172.21.15.152:0/1681359075 wait complete. 2024-04-02T04:55:42.961 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.955+0000 7fbaa4e29700 1 Processor -- start 2024-04-02T04:55:42.961 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.955+0000 7fbaa4e29700 1 -- start start 2024-04-02T04:55:42.961 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.955+0000 7fbaa4e29700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbaa00fbcd0 0x7fbaa00ffb70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:42.961 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.955+0000 7fbaa4e29700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbaa00fcf80 0x7fbaa00fe2c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:42.961 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.955+0000 7fbaa4e29700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbaa0100170 con 0x7fbaa00fbcd0 2024-04-02T04:55:42.961 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.955+0000 7fbaa4e29700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbaa00fe800 con 0x7fbaa00fcf80 2024-04-02T04:55:42.961 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.955+0000 7fba9e59c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbaa00fbcd0 0x7fbaa00ffb70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:42.961 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.955+0000 7fba9e59c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbaa00fbcd0 0x7fbaa00ffb70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:46326/0 (socket says 172.21.15.152:46326) 2024-04-02T04:55:42.961 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.955+0000 7fba9e59c700 1 -- 172.21.15.152:0/3630804515 learned_addr learned my addr 172.21.15.152:0/3630804515 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:55:42.961 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.956+0000 7fba9dd9b700 1 --2- 172.21.15.152:0/3630804515 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbaa00fcf80 0x7fbaa00fe2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:42.962 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.956+0000 7fba9e59c700 1 -- 172.21.15.152:0/3630804515 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbaa00fcf80 msgr2=0x7fbaa00fe2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:42.962 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.956+0000 7fba9e59c700 1 --2- 172.21.15.152:0/3630804515 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbaa00fcf80 0x7fbaa00fe2c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:42.962 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.956+0000 7fba9e59c700 1 -- 172.21.15.152:0/3630804515 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fba94005040 con 0x7fbaa00fbcd0 2024-04-02T04:55:42.962 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.956+0000 7fba9e59c700 1 --2- 172.21.15.152:0/3630804515 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbaa00fbcd0 0x7fbaa00ffb70 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7fba9400b970 tx=0x7fba94009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:42.962 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.956+0000 7fba8effd700 1 -- 172.21.15.152:0/3630804515 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fba9400e040 con 0x7fbaa00fbcd0 2024-04-02T04:55:42.962 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.956+0000 7fba8effd700 1 -- 172.21.15.152:0/3630804515 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fba9400bcc0 con 0x7fbaa00fbcd0 2024-04-02T04:55:42.962 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.956+0000 7fbaa4e29700 1 -- 172.21.15.152:0/3630804515 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbaa00fea80 con 0x7fbaa00fbcd0 2024-04-02T04:55:42.962 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.956+0000 7fba8effd700 1 -- 172.21.15.152:0/3630804515 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fba94013660 con 0x7fbaa00fbcd0 2024-04-02T04:55:42.962 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.956+0000 7fbaa4e29700 1 -- 172.21.15.152:0/3630804515 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbaa00ff040 con 0x7fbaa00fbcd0 2024-04-02T04:55:42.963 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.957+0000 7fbaa4e29700 1 -- 172.21.15.152:0/3630804515 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbaa004c710 con 0x7fbaa00fbcd0 2024-04-02T04:55:42.963 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.958+0000 7fba8effd700 1 -- 172.21.15.152:0/3630804515 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7fba9401ca00 con 0x7fbaa00fbcd0 2024-04-02T04:55:42.963 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.958+0000 7fba8effd700 1 --2- 172.21.15.152:0/3630804515 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fba880692a0 0x7fba8806b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:42.963 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.958+0000 7fba8effd700 1 -- 172.21.15.152:0/3630804515 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7fba94087b70 con 0x7fbaa00fbcd0 2024-04-02T04:55:42.963 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.958+0000 7fba9dd9b700 1 --2- 172.21.15.152:0/3630804515 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fba880692a0 0x7fba8806b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:42.964 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.958+0000 7fba9dd9b700 1 --2- 172.21.15.152:0/3630804515 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fba880692a0 0x7fba8806b750 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7fba900041d0 tx=0x7fba90009040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:42.966 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:42.962+0000 7fba8effd700 1 -- 172.21.15.152:0/3630804515 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fba94057d20 con 0x7fbaa00fbcd0 2024-04-02T04:55:43.160 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:43.156+0000 7fbaa4e29700 1 -- 172.21.15.152:0/3630804515 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "orch host ls", "target": ["mon-mgr", ""]}) v1 -- 0x7fbaa005e400 con 0x7fba880692a0 2024-04-02T04:55:43.161 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:43.158+0000 7fba8effd700 1 -- 172.21.15.152:0/3630804515 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+148 (secure 0 0 0) 0x7fbaa005e400 con 0x7fba880692a0 2024-04-02T04:55:43.162 INFO:teuthology.orchestra.run.smithi152.stdout:HOST ADDR LABELS STATUS 2024-04-02T04:55:43.162 INFO:teuthology.orchestra.run.smithi152.stdout:smithi152 172.21.15.152 2024-04-02T04:55:43.162 INFO:teuthology.orchestra.run.smithi152.stdout:smithi154 172.21.15.154 2024-04-02T04:55:43.162 INFO:teuthology.orchestra.run.smithi152.stdout:2 hosts in cluster 2024-04-02T04:55:43.164 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:43.160+0000 7fbaa4e29700 1 -- 172.21.15.152:0/3630804515 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fba880692a0 msgr2=0x7fba8806b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:43.164 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:43.160+0000 7fbaa4e29700 1 --2- 172.21.15.152:0/3630804515 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fba880692a0 0x7fba8806b750 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 crypto rx=0x7fba900041d0 tx=0x7fba90009040 comp rx=0 tx=0).stop 2024-04-02T04:55:43.164 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:43.160+0000 7fbaa4e29700 1 -- 172.21.15.152:0/3630804515 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbaa00fbcd0 msgr2=0x7fbaa00ffb70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:43.164 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:43.160+0000 7fbaa4e29700 1 --2- 172.21.15.152:0/3630804515 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbaa00fbcd0 0x7fbaa00ffb70 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 crypto rx=0x7fba9400b970 tx=0x7fba94009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:43.165 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:43.160+0000 7fbaa4e29700 1 -- 172.21.15.152:0/3630804515 shutdown_connections 2024-04-02T04:55:43.165 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:43.160+0000 7fbaa4e29700 1 --2- 172.21.15.152:0/3630804515 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fba880692a0 0x7fba8806b750 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:43.165 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:43.160+0000 7fbaa4e29700 1 --2- 172.21.15.152:0/3630804515 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbaa00fbcd0 0x7fbaa00ffb70 unknown :-1 s=CLOSED pgs=229 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:43.165 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:43.160+0000 7fbaa4e29700 1 --2- 172.21.15.152:0/3630804515 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbaa00fcf80 0x7fbaa00fe2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:43.165 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:43.160+0000 7fbaa4e29700 1 -- 172.21.15.152:0/3630804515 >> 172.21.15.152:0/3630804515 conn(0x7fbaa00f7450 msgr2=0x7fbaa0066810 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:43.165 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:43.160+0000 7fbaa4e29700 1 -- 172.21.15.152:0/3630804515 shutdown_connections 2024-04-02T04:55:43.165 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:43.160+0000 7fbaa4e29700 1 -- 172.21.15.152:0/3630804515 wait complete. 2024-04-02T04:55:43.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:43 smithi154 ceph-mon[35729]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:43.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:43 smithi152 ceph-mon[29073]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:43.814 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- bash -c 'ceph orch device ls' 2024-04-02T04:55:44.068 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:55:44.491 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:44 smithi152 ceph-mon[29073]: from='client.14444 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:55:44.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:44 smithi154 ceph-mon[35729]: from='client.14444 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:55:45.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:45 smithi152 ceph-mon[29073]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:45.708 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.702+0000 7f12b23d1700 1 -- 172.21.15.152:0/3304947921 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f12ac0fed90 msgr2=0x7f12ac0faed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:45.708 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.702+0000 7f12b23d1700 1 --2- 172.21.15.152:0/3304947921 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f12ac0fed90 0x7f12ac0faed0 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 crypto rx=0x7f129c004940 tx=0x7f129c009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:45.708 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.702+0000 7f12b23d1700 1 -- 172.21.15.152:0/3304947921 shutdown_connections 2024-04-02T04:55:45.708 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.702+0000 7f12b23d1700 1 --2- 172.21.15.152:0/3304947921 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f12ac0fed90 0x7f12ac0faed0 unknown :-1 s=CLOSED pgs=230 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:45.708 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.702+0000 7f12b23d1700 1 --2- 172.21.15.152:0/3304947921 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f12ac0fe440 0x7f12ac0fe850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:45.708 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.702+0000 7f12b23d1700 1 -- 172.21.15.152:0/3304947921 >> 172.21.15.152:0/3304947921 conn(0x7f12ac0f6be0 msgr2=0x7f12ac0f9010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:45.708 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.702+0000 7f12b23d1700 1 -- 172.21.15.152:0/3304947921 shutdown_connections 2024-04-02T04:55:45.709 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.702+0000 7f12b23d1700 1 -- 172.21.15.152:0/3304947921 wait complete. 2024-04-02T04:55:45.709 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.703+0000 7f12b23d1700 1 Processor -- start 2024-04-02T04:55:45.709 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.703+0000 7f12b23d1700 1 -- start start 2024-04-02T04:55:45.709 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.703+0000 7f12b23d1700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f12ac0fe440 0x7f12ac10d940 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:45.709 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.703+0000 7f12b23d1700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f12ac0fed90 0x7f12ac10de80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:45.709 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.703+0000 7f12b23d1700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f12ac10e510 con 0x7f12ac0fe440 2024-04-02T04:55:45.709 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.703+0000 7f12b23d1700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f12ac112eb0 con 0x7f12ac0fed90 2024-04-02T04:55:45.709 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.703+0000 7f12abfff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f12ac0fe440 0x7f12ac10d940 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:45.709 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.703+0000 7f12abfff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f12ac0fe440 0x7f12ac10d940 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:46354/0 (socket says 172.21.15.152:46354) 2024-04-02T04:55:45.709 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.703+0000 7f12abfff700 1 -- 172.21.15.152:0/175798716 learned_addr learned my addr 172.21.15.152:0/175798716 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:55:45.709 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.703+0000 7f12ab7fe700 1 --2- 172.21.15.152:0/175798716 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f12ac0fed90 0x7f12ac10de80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:45.709 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.704+0000 7f12abfff700 1 -- 172.21.15.152:0/175798716 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f12ac0fed90 msgr2=0x7f12ac10de80 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:45.709 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.704+0000 7f12abfff700 1 --2- 172.21.15.152:0/175798716 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f12ac0fed90 0x7f12ac10de80 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:45.710 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.704+0000 7f12abfff700 1 -- 172.21.15.152:0/175798716 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f129c005040 con 0x7f12ac0fe440 2024-04-02T04:55:45.710 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.704+0000 7f12abfff700 1 --2- 172.21.15.152:0/175798716 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f12ac0fe440 0x7f12ac10d940 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f12a000b790 tx=0x7f12a0009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:45.710 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.704+0000 7f12a8ff9700 1 -- 172.21.15.152:0/175798716 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f12a000f6f0 con 0x7f12ac0fe440 2024-04-02T04:55:45.710 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.704+0000 7f12b23d1700 1 -- 172.21.15.152:0/175798716 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f12ac113110 con 0x7f12ac0fe440 2024-04-02T04:55:45.710 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.704+0000 7f12a8ff9700 1 -- 172.21.15.152:0/175798716 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f12a000fd20 con 0x7f12ac0fe440 2024-04-02T04:55:45.710 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.704+0000 7f12a8ff9700 1 -- 172.21.15.152:0/175798716 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f12a00103a0 con 0x7f12ac0fe440 2024-04-02T04:55:45.710 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.704+0000 7f12b23d1700 1 -- 172.21.15.152:0/175798716 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f12ac113680 con 0x7f12ac0fe440 2024-04-02T04:55:45.711 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.705+0000 7f12b23d1700 1 -- 172.21.15.152:0/175798716 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f12ac04c710 con 0x7f12ac0fe440 2024-04-02T04:55:45.711 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.706+0000 7f12a8ff9700 1 -- 172.21.15.152:0/175798716 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f12a0011400 con 0x7f12ac0fe440 2024-04-02T04:55:45.711 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.706+0000 7f12a8ff9700 1 --2- 172.21.15.152:0/175798716 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f12940692a0 0x7f129406b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:45.711 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.706+0000 7f12a8ff9700 1 -- 172.21.15.152:0/175798716 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7f12a0086c30 con 0x7f12ac0fe440 2024-04-02T04:55:45.711 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.706+0000 7f12ab7fe700 1 --2- 172.21.15.152:0/175798716 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f12940692a0 0x7f129406b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:45.711 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.706+0000 7f12ab7fe700 1 --2- 172.21.15.152:0/175798716 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f12940692a0 0x7f129406b750 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f129c00b970 tx=0x7f129c009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:45.714 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.710+0000 7f12a8ff9700 1 -- 172.21.15.152:0/175798716 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f12a0056d30 con 0x7f12ac0fe440 2024-04-02T04:55:45.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:45 smithi154 ceph-mon[35729]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:45.908 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.904+0000 7f12b23d1700 1 -- 172.21.15.152:0/175798716 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "orch device ls", "target": ["mon-mgr", ""]}) v1 -- 0x7f12ac102490 con 0x7f12940692a0 2024-04-02T04:55:45.910 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.906+0000 7f12a8ff9700 1 -- 172.21.15.152:0/175798716 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+399 (secure 0 0 0) 0x7f12ac102490 con 0x7f12940692a0 2024-04-02T04:55:45.910 INFO:teuthology.orchestra.run.smithi152.stdout:HOST PATH TYPE DEVICE ID SIZE AVAILABLE REFRESHED REJECT REASONS 2024-04-02T04:55:45.910 INFO:teuthology.orchestra.run.smithi152.stdout:smithi152 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT623300JT400BGN 372G No 2m ago Has a FileSystem, LVM detected 2024-04-02T04:55:45.910 INFO:teuthology.orchestra.run.smithi152.stdout:smithi154 /dev/nvme0n1 ssd INTEL_SSDPEDMD400G4_CVFT6233003B400BGN 372G No 72s ago Has a FileSystem, LVM detected 2024-04-02T04:55:45.913 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.909+0000 7f12b23d1700 1 -- 172.21.15.152:0/175798716 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f12940692a0 msgr2=0x7f129406b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:45.913 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.909+0000 7f12b23d1700 1 --2- 172.21.15.152:0/175798716 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f12940692a0 0x7f129406b750 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 crypto rx=0x7f129c00b970 tx=0x7f129c009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:45.913 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.909+0000 7f12b23d1700 1 -- 172.21.15.152:0/175798716 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f12ac0fe440 msgr2=0x7f12ac10d940 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:45.913 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.909+0000 7f12b23d1700 1 --2- 172.21.15.152:0/175798716 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f12ac0fe440 0x7f12ac10d940 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 crypto rx=0x7f12a000b790 tx=0x7f12a0009250 comp rx=0 tx=0).stop 2024-04-02T04:55:45.913 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.909+0000 7f12b23d1700 1 -- 172.21.15.152:0/175798716 shutdown_connections 2024-04-02T04:55:45.913 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.909+0000 7f12b23d1700 1 --2- 172.21.15.152:0/175798716 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f12940692a0 0x7f129406b750 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:45.913 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.909+0000 7f12b23d1700 1 --2- 172.21.15.152:0/175798716 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f12ac0fe440 0x7f12ac10d940 unknown :-1 s=CLOSED pgs=231 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:45.913 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.909+0000 7f12b23d1700 1 --2- 172.21.15.152:0/175798716 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f12ac0fed90 0x7f12ac10de80 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:45.913 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.909+0000 7f12b23d1700 1 -- 172.21.15.152:0/175798716 >> 172.21.15.152:0/175798716 conn(0x7f12ac0f6be0 msgr2=0x7f12ac0f8210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:45.913 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.909+0000 7f12b23d1700 1 -- 172.21.15.152:0/175798716 shutdown_connections 2024-04-02T04:55:45.913 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:45.909+0000 7f12b23d1700 1 -- 172.21.15.152:0/175798716 wait complete. 2024-04-02T04:55:46.638 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-02T04:55:46.648 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi152.front.sepia.ceph.com 2024-04-02T04:55:46.648 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- bash -c 'ceph fs volume create cephfs --placement=4' 2024-04-02T04:55:46.903 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:55:47.678 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:47 smithi152 ceph-mon[29073]: from='client.14448 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:55:47.678 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:47 smithi152 ceph-mon[29073]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:47.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:47 smithi154 ceph-mon[35729]: from='client.14448 -' entity='client.admin' cmd=[{"prefix": "orch device ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:55:47.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:47 smithi154 ceph-mon[35729]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:48.491 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.485+0000 7f9ec2337700 1 -- 172.21.15.152:0/4114644788 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f9ebc066eb0 msgr2=0x7f9ebc0ffa80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:48.491 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.485+0000 7f9ec2337700 1 --2- 172.21.15.152:0/4114644788 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f9ebc066eb0 0x7f9ebc0ffa80 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 crypto rx=0x7f9ea4004940 tx=0x7f9ea4009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:48.491 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.485+0000 7f9ec2337700 1 -- 172.21.15.152:0/4114644788 shutdown_connections 2024-04-02T04:55:48.491 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.485+0000 7f9ec2337700 1 --2- 172.21.15.152:0/4114644788 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9ebc0fffc0 0x7f9ebc1023a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:48.491 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.485+0000 7f9ec2337700 1 --2- 172.21.15.152:0/4114644788 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f9ebc066eb0 0x7f9ebc0ffa80 unknown :-1 s=CLOSED pgs=232 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:48.491 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.485+0000 7f9ec2337700 1 -- 172.21.15.152:0/4114644788 >> 172.21.15.152:0/4114644788 conn(0x7f9ebc0f7430 msgr2=0x7f9ebc0f9860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:48.491 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.486+0000 7f9ec2337700 1 -- 172.21.15.152:0/4114644788 shutdown_connections 2024-04-02T04:55:48.491 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.486+0000 7f9ec2337700 1 -- 172.21.15.152:0/4114644788 wait complete. 2024-04-02T04:55:48.491 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.486+0000 7f9ec2337700 1 Processor -- start 2024-04-02T04:55:48.491 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.486+0000 7f9ec2337700 1 -- start start 2024-04-02T04:55:48.492 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.486+0000 7f9ec2337700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9ebc066eb0 0x7f9ebc10d9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:48.492 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.486+0000 7f9ec2337700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f9ebc0fffc0 0x7f9ebc10dee0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:48.492 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.486+0000 7f9ec2337700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9ebc10e570 con 0x7f9ebc0fffc0 2024-04-02T04:55:48.492 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.486+0000 7f9ec2337700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9ebc112f10 con 0x7f9ebc066eb0 2024-04-02T04:55:48.492 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.486+0000 7f9ebb7fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f9ebc0fffc0 0x7f9ebc10dee0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:48.492 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.487+0000 7f9ebb7fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f9ebc0fffc0 0x7f9ebc10dee0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:46370/0 (socket says 172.21.15.152:46370) 2024-04-02T04:55:48.492 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.487+0000 7f9ebb7fe700 1 -- 172.21.15.152:0/3575923278 learned_addr learned my addr 172.21.15.152:0/3575923278 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:55:48.492 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.487+0000 7f9ebbfff700 1 --2- 172.21.15.152:0/3575923278 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9ebc066eb0 0x7f9ebc10d9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:48.492 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.487+0000 7f9ebb7fe700 1 -- 172.21.15.152:0/3575923278 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9ebc066eb0 msgr2=0x7f9ebc10d9a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:48.492 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.487+0000 7f9ebb7fe700 1 --2- 172.21.15.152:0/3575923278 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9ebc066eb0 0x7f9ebc10d9a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:48.492 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.487+0000 7f9ebb7fe700 1 -- 172.21.15.152:0/3575923278 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9ea4005040 con 0x7f9ebc0fffc0 2024-04-02T04:55:48.492 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.487+0000 7f9ebb7fe700 1 --2- 172.21.15.152:0/3575923278 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f9ebc0fffc0 0x7f9ebc10dee0 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f9eac00b790 tx=0x7f9eac009220 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:48.492 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.487+0000 7f9eb8ff9700 1 -- 172.21.15.152:0/3575923278 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f9eac00f6f0 con 0x7f9ebc0fffc0 2024-04-02T04:55:48.492 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.487+0000 7f9ec2337700 1 -- 172.21.15.152:0/3575923278 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9ebc113170 con 0x7f9ebc0fffc0 2024-04-02T04:55:48.492 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.487+0000 7f9eb8ff9700 1 -- 172.21.15.152:0/3575923278 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f9eac00fd20 con 0x7f9ebc0fffc0 2024-04-02T04:55:48.493 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.487+0000 7f9eb8ff9700 1 -- 172.21.15.152:0/3575923278 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f9eac0103a0 con 0x7f9ebc0fffc0 2024-04-02T04:55:48.493 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.487+0000 7f9ec2337700 1 -- 172.21.15.152:0/3575923278 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9ebc1136e0 con 0x7f9ebc0fffc0 2024-04-02T04:55:48.493 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.488+0000 7f9ec2337700 1 -- 172.21.15.152:0/3575923278 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9ebc04c710 con 0x7f9ebc0fffc0 2024-04-02T04:55:48.493 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.489+0000 7f9eb8ff9700 1 -- 172.21.15.152:0/3575923278 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f9eac011400 con 0x7f9ebc0fffc0 2024-04-02T04:55:48.494 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.489+0000 7f9eb8ff9700 1 --2- 172.21.15.152:0/3575923278 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f9ea80692a0 0x7f9ea806b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:48.494 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.489+0000 7f9ebbfff700 1 --2- 172.21.15.152:0/3575923278 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f9ea80692a0 0x7f9ea806b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:48.494 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.489+0000 7f9eb8ff9700 1 -- 172.21.15.152:0/3575923278 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(40..40 src has 1..40) v4 ==== 4634+0+0 (secure 0 0 0) 0x7f9eac086c20 con 0x7f9ebc0fffc0 2024-04-02T04:55:48.494 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.490+0000 7f9ebbfff700 1 --2- 172.21.15.152:0/3575923278 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f9ea80692a0 0x7f9ea806b750 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f9ea4004890 tx=0x7f9ea4009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:48.497 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.494+0000 7f9eb8ff9700 1 -- 172.21.15.152:0/3575923278 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f9eac056d20 con 0x7f9ebc0fffc0 2024-04-02T04:55:48.713 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:48.709+0000 7f9ec2337700 1 -- 172.21.15.152:0/3575923278 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "fs volume create", "name": "cephfs", "placement": "4", "target": ["mon-mgr", ""]}) v1 -- 0x7f9ebc0f8f70 con 0x7f9ea80692a0 2024-04-02T04:55:49.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:49 smithi154 ceph-mon[35729]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:49.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:49 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd pool create", "pool": "cephfs.cephfs.meta"}]: dispatch 2024-04-02T04:55:49.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:49 smithi152 ceph-mon[29073]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:49.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:49 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd pool create", "pool": "cephfs.cephfs.meta"}]: dispatch 2024-04-02T04:55:50.467 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:50.464+0000 7f9eb8ff9700 1 -- 172.21.15.152:0/3575923278 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+0 (secure 0 0 0) 0x7f9ebc0f8f70 con 0x7f9ea80692a0 2024-04-02T04:55:50.472 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:50.468+0000 7f9ec2337700 1 -- 172.21.15.152:0/3575923278 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f9ea80692a0 msgr2=0x7f9ea806b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:50.472 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:50.468+0000 7f9ec2337700 1 --2- 172.21.15.152:0/3575923278 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f9ea80692a0 0x7f9ea806b750 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 crypto rx=0x7f9ea4004890 tx=0x7f9ea4009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:50.472 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:50.468+0000 7f9ec2337700 1 -- 172.21.15.152:0/3575923278 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f9ebc0fffc0 msgr2=0x7f9ebc10dee0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:50.472 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:50.468+0000 7f9ec2337700 1 --2- 172.21.15.152:0/3575923278 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f9ebc0fffc0 0x7f9ebc10dee0 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 crypto rx=0x7f9eac00b790 tx=0x7f9eac009220 comp rx=0 tx=0).stop 2024-04-02T04:55:50.472 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:50.468+0000 7f9ec2337700 1 -- 172.21.15.152:0/3575923278 shutdown_connections 2024-04-02T04:55:50.472 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:50.468+0000 7f9ec2337700 1 --2- 172.21.15.152:0/3575923278 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f9ea80692a0 0x7f9ea806b750 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:50.472 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:50.468+0000 7f9ec2337700 1 --2- 172.21.15.152:0/3575923278 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f9ebc066eb0 0x7f9ebc10d9a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:50.472 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:50.468+0000 7f9ec2337700 1 --2- 172.21.15.152:0/3575923278 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f9ebc0fffc0 0x7f9ebc10dee0 unknown :-1 s=CLOSED pgs=233 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:50.472 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:50.468+0000 7f9ec2337700 1 -- 172.21.15.152:0/3575923278 >> 172.21.15.152:0/3575923278 conn(0x7f9ebc0f7430 msgr2=0x7f9ebc0f7850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:50.472 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:50.469+0000 7f9ec2337700 1 -- 172.21.15.152:0/3575923278 shutdown_connections 2024-04-02T04:55:50.472 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:50.469+0000 7f9ec2337700 1 -- 172.21.15.152:0/3575923278 wait complete. 2024-04-02T04:55:50.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:50 smithi154 ceph-mon[35729]: from='client.14452 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "cephfs", "placement": "4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:55:50.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:50 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd='[{"prefix": "osd pool create", "pool": "cephfs.cephfs.meta"}]': finished 2024-04-02T04:55:50.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:50 smithi154 ceph-mon[35729]: osdmap e41: 6 total, 6 up, 6 in 2024-04-02T04:55:50.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:50 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.cephfs.data"}]: dispatch 2024-04-02T04:55:50.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:50 smithi152 ceph-mon[29073]: from='client.14452 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "cephfs", "placement": "4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:55:50.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:50 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd='[{"prefix": "osd pool create", "pool": "cephfs.cephfs.meta"}]': finished 2024-04-02T04:55:50.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:50 smithi152 ceph-mon[29073]: osdmap e41: 6 total, 6 up, 6 in 2024-04-02T04:55:50.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:50 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.cephfs.data"}]: dispatch 2024-04-02T04:55:50.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:50 smithi152 ceph-2e0055d0-f0ac-11ee-b647-cb9ed24678a4-mon-smithi152[29069]: 2024-04-02T04:55:50.445+0000 7fd1f4da1700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-02T04:55:51.051 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- bash -c 'ceph fs dump' 2024-04-02T04:55:51.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:51 smithi154 ceph-mon[35729]: pgmap v169: 33 pgs: 5 creating+peering, 1 active+clean, 27 unknown; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:51.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:51 smithi154 ceph-mon[35729]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-02T04:55:51.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:51 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.cephfs.data"}]': finished 2024-04-02T04:55:51.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:51 smithi154 ceph-mon[35729]: osdmap e42: 6 total, 6 up, 6 in 2024-04-02T04:55:51.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:51 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs.cephfs.meta", "data": "cephfs.cephfs.data"}]: dispatch 2024-04-02T04:55:51.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:51 smithi154 ceph-mon[35729]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-02T04:55:51.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:51 smithi154 ceph-mon[35729]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-02T04:55:51.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:51 smithi154 ceph-mon[35729]: osdmap e43: 6 total, 6 up, 6 in 2024-04-02T04:55:51.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:51 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs.cephfs.meta", "data": "cephfs.cephfs.data"}]': finished 2024-04-02T04:55:51.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:51 smithi154 ceph-mon[35729]: fsmap cephfs:0 2024-04-02T04:55:51.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:51 smithi154 ceph-mon[35729]: Saving service mds.cephfs spec with placement count:4 2024-04-02T04:55:51.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:51 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:55:51.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:51 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:55:51.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:51 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:55:51.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:51 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:55:51.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:51 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:55:51.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:51 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get-or-create", "entity": "mds.cephfs.smithi152.lluqrs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-02T04:55:51.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:51 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd='[{"prefix": "auth get-or-create", "entity": "mds.cephfs.smithi152.lluqrs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-02T04:55:51.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:51 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:55:51.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:51 smithi152 ceph-mon[29073]: pgmap v169: 33 pgs: 5 creating+peering, 1 active+clean, 27 unknown; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:51.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:51 smithi152 ceph-mon[29073]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-02T04:55:51.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:51 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.cephfs.data"}]': finished 2024-04-02T04:55:51.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:51 smithi152 ceph-mon[29073]: osdmap e42: 6 total, 6 up, 6 in 2024-04-02T04:55:51.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:51 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs.cephfs.meta", "data": "cephfs.cephfs.data"}]: dispatch 2024-04-02T04:55:51.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:51 smithi152 ceph-mon[29073]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-02T04:55:51.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:51 smithi152 ceph-mon[29073]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-02T04:55:51.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:51 smithi152 ceph-mon[29073]: osdmap e43: 6 total, 6 up, 6 in 2024-04-02T04:55:51.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:51 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs.cephfs.meta", "data": "cephfs.cephfs.data"}]': finished 2024-04-02T04:55:51.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:51 smithi152 ceph-mon[29073]: fsmap cephfs:0 2024-04-02T04:55:51.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:51 smithi152 ceph-mon[29073]: Saving service mds.cephfs spec with placement count:4 2024-04-02T04:55:51.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:51 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:55:51.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:51 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:55:51.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:51 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:55:51.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:51 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:55:51.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:51 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:55:51.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:51 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get-or-create", "entity": "mds.cephfs.smithi152.lluqrs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-02T04:55:51.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:51 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd='[{"prefix": "auth get-or-create", "entity": "mds.cephfs.smithi152.lluqrs", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-02T04:55:51.759 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:51 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:55:52.359 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:55:52.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:52 smithi154 ceph-mon[35729]: Deploying daemon mds.cephfs.smithi152.lluqrs on smithi152 2024-04-02T04:55:52.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:52 smithi154 ceph-mon[35729]: osdmap e44: 6 total, 6 up, 6 in 2024-04-02T04:55:52.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:52 smithi152 ceph-mon[29073]: Deploying daemon mds.cephfs.smithi152.lluqrs on smithi152 2024-04-02T04:55:52.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:52 smithi152 ceph-mon[29073]: osdmap e44: 6 total, 6 up, 6 in 2024-04-02T04:55:53.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:53 smithi154 ceph-mon[35729]: pgmap v173: 65 pgs: 4 creating+activating, 5 creating+peering, 9 active+clean, 47 unknown; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:53.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:53 smithi154 ceph-mon[35729]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-02T04:55:53.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:53 smithi154 ceph-mon[35729]: osdmap e45: 6 total, 6 up, 6 in 2024-04-02T04:55:53.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:53 smithi152 ceph-mon[29073]: pgmap v173: 65 pgs: 4 creating+activating, 5 creating+peering, 9 active+clean, 47 unknown; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:53.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:53 smithi152 ceph-mon[29073]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-02T04:55:53.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:53 smithi152 ceph-mon[29073]: osdmap e45: 6 total, 6 up, 6 in 2024-04-02T04:55:54.505 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.498+0000 7fb813f88700 1 -- 172.21.15.152:0/3502549432 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb80c0fbcf0 msgr2=0x7fb80c0fc100 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:54.505 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.498+0000 7fb813f88700 1 --2- 172.21.15.152:0/3502549432 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb80c0fbcf0 0x7fb80c0fc100 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 crypto rx=0x7fb804009ce0 tx=0x7fb804004d10 comp rx=0 tx=0).stop 2024-04-02T04:55:54.506 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.498+0000 7fb813f88700 1 -- 172.21.15.152:0/3502549432 shutdown_connections 2024-04-02T04:55:54.506 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.498+0000 7fb813f88700 1 --2- 172.21.15.152:0/3502549432 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb80c0fcfa0 0x7fb80c0fd410 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:54.506 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.498+0000 7fb813f88700 1 --2- 172.21.15.152:0/3502549432 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb80c0fbcf0 0x7fb80c0fc100 unknown :-1 s=CLOSED pgs=236 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:54.506 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.498+0000 7fb813f88700 1 -- 172.21.15.152:0/3502549432 >> 172.21.15.152:0/3502549432 conn(0x7fb80c0f7450 msgr2=0x7fb80c0f9880 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:54.506 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.499+0000 7fb813f88700 1 -- 172.21.15.152:0/3502549432 shutdown_connections 2024-04-02T04:55:54.506 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.499+0000 7fb813f88700 1 -- 172.21.15.152:0/3502549432 wait complete. 2024-04-02T04:55:54.506 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.499+0000 7fb813f88700 1 Processor -- start 2024-04-02T04:55:54.506 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.499+0000 7fb813f88700 1 -- start start 2024-04-02T04:55:54.506 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.499+0000 7fb813f88700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb80c0fbcf0 0x7fb80c10ca90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:54.506 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.499+0000 7fb813f88700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb80c0fcfa0 0x7fb80c10cfd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:54.506 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.499+0000 7fb813f88700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb80c10d5d0 con 0x7fb80c0fcfa0 2024-04-02T04:55:54.506 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.499+0000 7fb813f88700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb80c10de20 con 0x7fb80c0fbcf0 2024-04-02T04:55:54.506 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.499+0000 7fb811523700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb80c0fcfa0 0x7fb80c10cfd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:54.507 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.500+0000 7fb811523700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb80c0fcfa0 0x7fb80c10cfd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:49978/0 (socket says 172.21.15.152:49978) 2024-04-02T04:55:54.507 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.500+0000 7fb811523700 1 -- 172.21.15.152:0/3300164171 learned_addr learned my addr 172.21.15.152:0/3300164171 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:55:54.507 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.500+0000 7fb811d24700 1 --2- 172.21.15.152:0/3300164171 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb80c0fbcf0 0x7fb80c10ca90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:54.507 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.500+0000 7fb811523700 1 -- 172.21.15.152:0/3300164171 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb80c0fbcf0 msgr2=0x7fb80c10ca90 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:54.507 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.500+0000 7fb811523700 1 --2- 172.21.15.152:0/3300164171 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb80c0fbcf0 0x7fb80c10ca90 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:54.507 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.500+0000 7fb811523700 1 -- 172.21.15.152:0/3300164171 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb804009940 con 0x7fb80c0fcfa0 2024-04-02T04:55:54.507 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.500+0000 7fb811523700 1 --2- 172.21.15.152:0/3300164171 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb80c0fcfa0 0x7fb80c10cfd0 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7fb7fc002a50 tx=0x7fb7fc0046b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:54.507 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.500+0000 7fb8027fc700 1 -- 172.21.15.152:0/3300164171 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fb7fc007cc0 con 0x7fb80c0fcfa0 2024-04-02T04:55:54.507 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.500+0000 7fb813f88700 1 -- 172.21.15.152:0/3300164171 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb80c10e100 con 0x7fb80c0fcfa0 2024-04-02T04:55:54.507 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.500+0000 7fb8027fc700 1 -- 172.21.15.152:0/3300164171 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fb7fc00f040 con 0x7fb80c0fcfa0 2024-04-02T04:55:54.507 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.500+0000 7fb8027fc700 1 -- 172.21.15.152:0/3300164171 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fb7fc00da30 con 0x7fb80c0fcfa0 2024-04-02T04:55:54.507 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.500+0000 7fb813f88700 1 -- 172.21.15.152:0/3300164171 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb80c10e640 con 0x7fb80c0fcfa0 2024-04-02T04:55:54.507 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.501+0000 7fb813f88700 1 -- 172.21.15.152:0/3300164171 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb80c05f5f0 con 0x7fb80c0fcfa0 2024-04-02T04:55:54.507 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.502+0000 7fb8027fc700 1 -- 172.21.15.152:0/3300164171 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7fb7fc00db90 con 0x7fb80c0fcfa0 2024-04-02T04:55:54.508 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.502+0000 7fb8027fc700 1 --2- 172.21.15.152:0/3300164171 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb7f80692a0 0x7fb7f806b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:54.508 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.502+0000 7fb8027fc700 1 -- 172.21.15.152:0/3300164171 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(45..45 src has 1..45) v4 ==== 5464+0+0 (secure 0 0 0) 0x7fb7fc086f40 con 0x7fb80c0fcfa0 2024-04-02T04:55:54.508 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.502+0000 7fb811d24700 1 --2- 172.21.15.152:0/3300164171 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb7f80692a0 0x7fb7f806b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:54.508 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.503+0000 7fb811d24700 1 --2- 172.21.15.152:0/3300164171 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb7f80692a0 0x7fb7f806b750 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7fb804015040 tx=0x7fb804004d10 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:54.510 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.507+0000 7fb8027fc700 1 -- 172.21.15.152:0/3300164171 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fb7fc056c80 con 0x7fb80c0fcfa0 2024-04-02T04:55:54.737 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.733+0000 7fb813f88700 1 -- 172.21.15.152:0/3300164171 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "fs dump"} v 0) v1 -- 0x7fb80c10e9c0 con 0x7fb80c0fcfa0 2024-04-02T04:55:54.737 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.733+0000 7fb8027fc700 1 -- 172.21.15.152:0/3300164171 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump"}]=0 dumped fsmap epoch 4 v4) v1 ==== 75+0+1249 (secure 0 0 0) 0x7fb7fc056810 con 0x7fb80c0fcfa0 2024-04-02T04:55:54.737 INFO:teuthology.orchestra.run.smithi152.stdout:e4 2024-04-02T04:55:54.738 INFO:teuthology.orchestra.run.smithi152.stdout:enable_multiple, ever_enabled_multiple: 1,1 2024-04-02T04:55:54.738 INFO:teuthology.orchestra.run.smithi152.stdout:default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-02T04:55:54.738 INFO:teuthology.orchestra.run.smithi152.stdout:legacy client fscid: 1 2024-04-02T04:55:54.738 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:55:54.738 INFO:teuthology.orchestra.run.smithi152.stdout:Filesystem 'cephfs' (1) 2024-04-02T04:55:54.738 INFO:teuthology.orchestra.run.smithi152.stdout:fs_name cephfs 2024-04-02T04:55:54.738 INFO:teuthology.orchestra.run.smithi152.stdout:epoch 4 2024-04-02T04:55:54.738 INFO:teuthology.orchestra.run.smithi152.stdout:flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-02T04:55:54.738 INFO:teuthology.orchestra.run.smithi152.stdout:created 2024-04-02T04:55:50.446442+0000 2024-04-02T04:55:54.738 INFO:teuthology.orchestra.run.smithi152.stdout:modified 2024-04-02T04:55:54.464489+0000 2024-04-02T04:55:54.738 INFO:teuthology.orchestra.run.smithi152.stdout:tableserver 0 2024-04-02T04:55:54.739 INFO:teuthology.orchestra.run.smithi152.stdout:root 0 2024-04-02T04:55:54.739 INFO:teuthology.orchestra.run.smithi152.stdout:session_timeout 60 2024-04-02T04:55:54.739 INFO:teuthology.orchestra.run.smithi152.stdout:session_autoclose 300 2024-04-02T04:55:54.739 INFO:teuthology.orchestra.run.smithi152.stdout:max_file_size 1099511627776 2024-04-02T04:55:54.739 INFO:teuthology.orchestra.run.smithi152.stdout:required_client_features {} 2024-04-02T04:55:54.739 INFO:teuthology.orchestra.run.smithi152.stdout:last_failure 0 2024-04-02T04:55:54.739 INFO:teuthology.orchestra.run.smithi152.stdout:last_failure_osd_epoch 0 2024-04-02T04:55:54.739 INFO:teuthology.orchestra.run.smithi152.stdout:compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-02T04:55:54.739 INFO:teuthology.orchestra.run.smithi152.stdout:max_mds 1 2024-04-02T04:55:54.739 INFO:teuthology.orchestra.run.smithi152.stdout:in 0 2024-04-02T04:55:54.739 INFO:teuthology.orchestra.run.smithi152.stdout:up {0=14456} 2024-04-02T04:55:54.739 INFO:teuthology.orchestra.run.smithi152.stdout:failed 2024-04-02T04:55:54.739 INFO:teuthology.orchestra.run.smithi152.stdout:damaged 2024-04-02T04:55:54.739 INFO:teuthology.orchestra.run.smithi152.stdout:stopped 2024-04-02T04:55:54.739 INFO:teuthology.orchestra.run.smithi152.stdout:data_pools [3] 2024-04-02T04:55:54.739 INFO:teuthology.orchestra.run.smithi152.stdout:metadata_pool 2 2024-04-02T04:55:54.740 INFO:teuthology.orchestra.run.smithi152.stdout:inline_data disabled 2024-04-02T04:55:54.740 INFO:teuthology.orchestra.run.smithi152.stdout:balancer 2024-04-02T04:55:54.740 INFO:teuthology.orchestra.run.smithi152.stdout:standby_count_wanted 0 2024-04-02T04:55:54.740 INFO:teuthology.orchestra.run.smithi152.stdout:[mds.cephfs.smithi152.lluqrs{0:14456} state up:creating seq 1 addr [v2:172.21.15.152:6826/4038990242,v1:172.21.15.152:6827/4038990242] compat {c=[1],r=[1],i=[7ff]}] 2024-04-02T04:55:54.740 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:55:54.740 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:55:54.741 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.736+0000 7fb813f88700 1 -- 172.21.15.152:0/3300164171 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb7f80692a0 msgr2=0x7fb7f806b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:54.741 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.736+0000 7fb813f88700 1 --2- 172.21.15.152:0/3300164171 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb7f80692a0 0x7fb7f806b750 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 crypto rx=0x7fb804015040 tx=0x7fb804004d10 comp rx=0 tx=0).stop 2024-04-02T04:55:54.741 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.736+0000 7fb813f88700 1 -- 172.21.15.152:0/3300164171 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb80c0fcfa0 msgr2=0x7fb80c10cfd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:54.741 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.736+0000 7fb813f88700 1 --2- 172.21.15.152:0/3300164171 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb80c0fcfa0 0x7fb80c10cfd0 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 crypto rx=0x7fb7fc002a50 tx=0x7fb7fc0046b0 comp rx=0 tx=0).stop 2024-04-02T04:55:54.741 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.736+0000 7fb813f88700 1 -- 172.21.15.152:0/3300164171 shutdown_connections 2024-04-02T04:55:54.741 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.736+0000 7fb813f88700 1 --2- 172.21.15.152:0/3300164171 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb7f80692a0 0x7fb7f806b750 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:54.741 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.736+0000 7fb813f88700 1 --2- 172.21.15.152:0/3300164171 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb80c0fbcf0 0x7fb80c10ca90 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:54.741 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.736+0000 7fb813f88700 1 --2- 172.21.15.152:0/3300164171 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb80c0fcfa0 0x7fb80c10cfd0 unknown :-1 s=CLOSED pgs=237 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:54.741 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.736+0000 7fb813f88700 1 -- 172.21.15.152:0/3300164171 >> 172.21.15.152:0/3300164171 conn(0x7fb80c0f7450 msgr2=0x7fb80c0f8b60 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:54.741 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.737+0000 7fb813f88700 1 -- 172.21.15.152:0/3300164171 shutdown_connections 2024-04-02T04:55:54.741 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:54.737+0000 7fb813f88700 1 -- 172.21.15.152:0/3300164171 wait complete. 2024-04-02T04:55:54.742 INFO:teuthology.orchestra.run.smithi152.stderr:dumped fsmap epoch 4 2024-04-02T04:55:55.427 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:55 smithi152 ceph-mon[29073]: pgmap v175: 65 pgs: 4 creating+activating, 5 creating+peering, 52 active+clean, 4 unknown; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:55.427 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:55 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:55:55.427 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:55 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:55:55.427 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:55 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:55:55.427 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:55 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get-or-create", "entity": "mds.cephfs.smithi154.llvaoh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-02T04:55:55.427 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:55 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd='[{"prefix": "auth get-or-create", "entity": "mds.cephfs.smithi154.llvaoh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-02T04:55:55.427 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:55 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:55:55.427 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:55 smithi152 ceph-mon[29073]: Deploying daemon mds.cephfs.smithi154.llvaoh on smithi154 2024-04-02T04:55:55.427 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:55 smithi152 ceph-mon[29073]: mds.? [v2:172.21.15.152:6826/4038990242,v1:172.21.15.152:6827/4038990242] up:boot 2024-04-02T04:55:55.427 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:55 smithi152 ceph-mon[29073]: daemon mds.cephfs.smithi152.lluqrs assigned to filesystem cephfs as rank 0 (now has 1 ranks) 2024-04-02T04:55:55.428 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:55 smithi152 ceph-mon[29073]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-02T04:55:55.428 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:55 smithi152 ceph-mon[29073]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-02T04:55:55.428 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:55 smithi152 ceph-mon[29073]: Cluster is now healthy 2024-04-02T04:55:55.428 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:55 smithi152 ceph-mon[29073]: fsmap cephfs:0 1 up:standby 2024-04-02T04:55:55.428 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:55 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mds metadata", "who": "cephfs.smithi152.lluqrs"}]: dispatch 2024-04-02T04:55:55.428 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:55 smithi152 ceph-mon[29073]: fsmap cephfs:1 {0=cephfs.smithi152.lluqrs=up:creating} 2024-04-02T04:55:55.428 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:55 smithi152 ceph-mon[29073]: daemon mds.cephfs.smithi152.lluqrs is now active in filesystem cephfs as rank 0 2024-04-02T04:55:55.428 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:55 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/3300164171' entity='client.admin' cmd=[{"prefix": "fs dump"}]: dispatch 2024-04-02T04:55:55.428 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:55 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:55:55.452 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-02T04:55:55.464 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi152.front.sepia.ceph.com 2024-04-02T04:55:55.465 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- bash -c 'ceph fs set cephfs max_mds 2' 2024-04-02T04:55:55.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:55 smithi154 ceph-mon[35729]: pgmap v175: 65 pgs: 4 creating+activating, 5 creating+peering, 52 active+clean, 4 unknown; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-04-02T04:55:55.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:55 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:55:55.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:55 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:55:55.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:55 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:55:55.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:55 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get-or-create", "entity": "mds.cephfs.smithi154.llvaoh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-02T04:55:55.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:55 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd='[{"prefix": "auth get-or-create", "entity": "mds.cephfs.smithi154.llvaoh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-02T04:55:55.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:55 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:55:55.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:55 smithi154 ceph-mon[35729]: Deploying daemon mds.cephfs.smithi154.llvaoh on smithi154 2024-04-02T04:55:55.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:55 smithi154 ceph-mon[35729]: mds.? [v2:172.21.15.152:6826/4038990242,v1:172.21.15.152:6827/4038990242] up:boot 2024-04-02T04:55:55.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:55 smithi154 ceph-mon[35729]: daemon mds.cephfs.smithi152.lluqrs assigned to filesystem cephfs as rank 0 (now has 1 ranks) 2024-04-02T04:55:55.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:55 smithi154 ceph-mon[35729]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-02T04:55:55.485 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:55 smithi154 ceph-mon[35729]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-02T04:55:55.485 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:55 smithi154 ceph-mon[35729]: Cluster is now healthy 2024-04-02T04:55:55.485 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:55 smithi154 ceph-mon[35729]: fsmap cephfs:0 1 up:standby 2024-04-02T04:55:55.485 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:55 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mds metadata", "who": "cephfs.smithi152.lluqrs"}]: dispatch 2024-04-02T04:55:55.485 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:55 smithi154 ceph-mon[35729]: fsmap cephfs:1 {0=cephfs.smithi152.lluqrs=up:creating} 2024-04-02T04:55:55.485 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:55 smithi154 ceph-mon[35729]: daemon mds.cephfs.smithi152.lluqrs is now active in filesystem cephfs as rank 0 2024-04-02T04:55:55.485 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:55 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/3300164171' entity='client.admin' cmd=[{"prefix": "fs dump"}]: dispatch 2024-04-02T04:55:55.485 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:55 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:55:55.731 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:55:56.606 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:56 smithi154 ceph-mon[35729]: mds.? [v2:172.21.15.152:6826/4038990242,v1:172.21.15.152:6827/4038990242] up:active 2024-04-02T04:55:56.606 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:56 smithi154 ceph-mon[35729]: fsmap cephfs:1 {0=cephfs.smithi152.lluqrs=up:active} 2024-04-02T04:55:56.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:56 smithi152 ceph-mon[29073]: mds.? [v2:172.21.15.152:6826/4038990242,v1:172.21.15.152:6827/4038990242] up:active 2024-04-02T04:55:56.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:56 smithi152 ceph-mon[29073]: fsmap cephfs:1 {0=cephfs.smithi152.lluqrs=up:active} 2024-04-02T04:55:57.359 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.353+0000 7fafac9ec700 1 -- 172.21.15.152:0/297569072 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fafa80fdfe0 msgr2=0x7fafa80fe450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:57.359 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.353+0000 7fafac9ec700 1 --2- 172.21.15.152:0/297569072 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fafa80fdfe0 0x7fafa80fe450 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 crypto rx=0x7faf9c004940 tx=0x7faf9c009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:57.360 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.354+0000 7fafac9ec700 1 -- 172.21.15.152:0/297569072 shutdown_connections 2024-04-02T04:55:57.360 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.354+0000 7fafac9ec700 1 --2- 172.21.15.152:0/297569072 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fafa80fdfe0 0x7fafa80fe450 unknown :-1 s=CLOSED pgs=238 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:57.360 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.354+0000 7fafac9ec700 1 --2- 172.21.15.152:0/297569072 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fafa80fce70 0x7fafa80fd280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:57.360 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.354+0000 7fafac9ec700 1 -- 172.21.15.152:0/297569072 >> 172.21.15.152:0/297569072 conn(0x7fafa80f8550 msgr2=0x7fafa80fa960 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:57.360 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.354+0000 7fafac9ec700 1 -- 172.21.15.152:0/297569072 shutdown_connections 2024-04-02T04:55:57.360 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.354+0000 7fafac9ec700 1 -- 172.21.15.152:0/297569072 wait complete. 2024-04-02T04:55:57.360 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.354+0000 7fafac9ec700 1 Processor -- start 2024-04-02T04:55:57.360 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.354+0000 7fafac9ec700 1 -- start start 2024-04-02T04:55:57.360 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.354+0000 7fafac9ec700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fafa80fce70 0x7fafa810d930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:57.360 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.354+0000 7fafac9ec700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fafa80fdfe0 0x7fafa810de70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:57.360 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.354+0000 7fafac9ec700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fafa810e500 con 0x7fafa80fce70 2024-04-02T04:55:57.360 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.354+0000 7fafac9ec700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fafa8112ea0 con 0x7fafa80fdfe0 2024-04-02T04:55:57.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.354+0000 7fafa659c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fafa80fce70 0x7fafa810d930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:57.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.354+0000 7fafa659c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fafa80fce70 0x7fafa810d930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:49988/0 (socket says 172.21.15.152:49988) 2024-04-02T04:55:57.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.354+0000 7fafa659c700 1 -- 172.21.15.152:0/2083062474 learned_addr learned my addr 172.21.15.152:0/2083062474 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:55:57.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.355+0000 7fafa5d9b700 1 --2- 172.21.15.152:0/2083062474 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fafa80fdfe0 0x7fafa810de70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:57.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.355+0000 7fafa659c700 1 -- 172.21.15.152:0/2083062474 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fafa80fdfe0 msgr2=0x7fafa810de70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:57.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.355+0000 7fafa659c700 1 --2- 172.21.15.152:0/2083062474 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fafa80fdfe0 0x7fafa810de70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:57.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.355+0000 7fafa659c700 1 -- 172.21.15.152:0/2083062474 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faf9c005040 con 0x7fafa80fce70 2024-04-02T04:55:57.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.355+0000 7fafa659c700 1 --2- 172.21.15.152:0/2083062474 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fafa80fce70 0x7fafa810d930 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7faf9800b790 tx=0x7faf98009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:57.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.355+0000 7faf96ffd700 1 -- 172.21.15.152:0/2083062474 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7faf9800f770 con 0x7fafa80fce70 2024-04-02T04:55:57.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.355+0000 7faf96ffd700 1 -- 172.21.15.152:0/2083062474 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7faf9800fda0 con 0x7fafa80fce70 2024-04-02T04:55:57.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.355+0000 7fafac9ec700 1 -- 172.21.15.152:0/2083062474 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fafa8113100 con 0x7fafa80fce70 2024-04-02T04:55:57.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.355+0000 7faf96ffd700 1 -- 172.21.15.152:0/2083062474 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7faf9800d310 con 0x7fafa80fce70 2024-04-02T04:55:57.361 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.355+0000 7fafac9ec700 1 -- 172.21.15.152:0/2083062474 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fafa8113670 con 0x7fafa80fce70 2024-04-02T04:55:57.362 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.356+0000 7fafac9ec700 1 -- 172.21.15.152:0/2083062474 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fafa804c710 con 0x7fafa80fce70 2024-04-02T04:55:57.362 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.356+0000 7faf96ffd700 1 -- 172.21.15.152:0/2083062474 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7faf98010420 con 0x7fafa80fce70 2024-04-02T04:55:57.362 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.357+0000 7faf96ffd700 1 --2- 172.21.15.152:0/2083062474 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7faf900692a0 0x7faf9006b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:55:57.362 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.357+0000 7faf96ffd700 1 -- 172.21.15.152:0/2083062474 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(45..45 src has 1..45) v4 ==== 5464+0+0 (secure 0 0 0) 0x7faf980871b0 con 0x7fafa80fce70 2024-04-02T04:55:57.363 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.357+0000 7fafa5d9b700 1 --2- 172.21.15.152:0/2083062474 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7faf900692a0 0x7faf9006b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:55:57.363 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.357+0000 7fafa5d9b700 1 --2- 172.21.15.152:0/2083062474 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7faf900692a0 0x7faf9006b750 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7faf9c00b3b0 tx=0x7faf9c009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:55:57.365 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.361+0000 7faf96ffd700 1 -- 172.21.15.152:0/2083062474 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7faf98056f70 con 0x7fafa80fce70 2024-04-02T04:55:57.612 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:57.608+0000 7fafac9ec700 1 -- 172.21.15.152:0/2083062474 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "2"} v 0) v1 -- 0x7fafa8070430 con 0x7fafa80fce70 2024-04-02T04:55:57.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:57 smithi154 ceph-mon[35729]: pgmap v176: 65 pgs: 4 creating+activating, 61 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 0 B/s wr, 2 op/s 2024-04-02T04:55:57.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:57 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:55:57.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:57 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:55:57.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:57 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:55:57.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:57 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get-or-create", "entity": "mds.cephfs.smithi152.ywghzy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-02T04:55:57.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:57 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd='[{"prefix": "auth get-or-create", "entity": "mds.cephfs.smithi152.ywghzy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-02T04:55:57.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:57 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:55:57.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:57 smithi152 ceph-mon[29073]: pgmap v176: 65 pgs: 4 creating+activating, 61 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 0 B/s wr, 2 op/s 2024-04-02T04:55:57.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:57 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:55:57.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:57 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:55:57.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:57 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:55:57.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:57 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get-or-create", "entity": "mds.cephfs.smithi152.ywghzy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-02T04:55:57.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:57 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd='[{"prefix": "auth get-or-create", "entity": "mds.cephfs.smithi152.ywghzy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-02T04:55:57.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:57 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:55:58.496 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:58.492+0000 7faf96ffd700 1 -- 172.21.15.152:0/2083062474 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "2"}]=0 v8) v1 ==== 105+0+0 (secure 0 0 0) 0x7faf98056b00 con 0x7fafa80fce70 2024-04-02T04:55:58.498 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:58.495+0000 7fafac9ec700 1 -- 172.21.15.152:0/2083062474 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7faf900692a0 msgr2=0x7faf9006b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:58.499 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:58.495+0000 7fafac9ec700 1 --2- 172.21.15.152:0/2083062474 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7faf900692a0 0x7faf9006b750 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 crypto rx=0x7faf9c00b3b0 tx=0x7faf9c009e30 comp rx=0 tx=0).stop 2024-04-02T04:55:58.499 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:58.495+0000 7fafac9ec700 1 -- 172.21.15.152:0/2083062474 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fafa80fce70 msgr2=0x7fafa810d930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:55:58.499 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:58.495+0000 7fafac9ec700 1 --2- 172.21.15.152:0/2083062474 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fafa80fce70 0x7fafa810d930 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 crypto rx=0x7faf9800b790 tx=0x7faf98009250 comp rx=0 tx=0).stop 2024-04-02T04:55:58.499 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:58.495+0000 7fafac9ec700 1 -- 172.21.15.152:0/2083062474 shutdown_connections 2024-04-02T04:55:58.499 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:58.495+0000 7fafac9ec700 1 --2- 172.21.15.152:0/2083062474 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7faf900692a0 0x7faf9006b750 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:58.499 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:58.495+0000 7fafac9ec700 1 --2- 172.21.15.152:0/2083062474 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fafa80fce70 0x7fafa810d930 unknown :-1 s=CLOSED pgs=239 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:58.499 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:58.495+0000 7fafac9ec700 1 --2- 172.21.15.152:0/2083062474 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fafa80fdfe0 0x7fafa810de70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:55:58.499 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:58.495+0000 7fafac9ec700 1 -- 172.21.15.152:0/2083062474 >> 172.21.15.152:0/2083062474 conn(0x7fafa80f8550 msgr2=0x7fafa80f9b80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:55:58.499 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:58.495+0000 7fafac9ec700 1 -- 172.21.15.152:0/2083062474 shutdown_connections 2024-04-02T04:55:58.499 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:55:58.495+0000 7fafac9ec700 1 -- 172.21.15.152:0/2083062474 wait complete. 2024-04-02T04:55:58.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:58 smithi152 ceph-mon[29073]: Deploying daemon mds.cephfs.smithi152.ywghzy on smithi152 2024-04-02T04:55:58.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:58 smithi152 ceph-mon[29073]: mds.? [v2:172.21.15.154:6824/1663663935,v1:172.21.15.154:6825/1663663935] up:boot 2024-04-02T04:55:58.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:58 smithi152 ceph-mon[29073]: fsmap cephfs:1 {0=cephfs.smithi152.lluqrs=up:active} 1 up:standby 2024-04-02T04:55:58.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:58 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mds metadata", "who": "cephfs.smithi154.llvaoh"}]: dispatch 2024-04-02T04:55:58.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:58 smithi152 ceph-mon[29073]: fsmap cephfs:1 {0=cephfs.smithi152.lluqrs=up:active} 1 up:standby 2024-04-02T04:55:58.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:58 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/2083062474' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "2"}]: dispatch 2024-04-02T04:55:58.872 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:58 smithi154 ceph-mon[35729]: Deploying daemon mds.cephfs.smithi152.ywghzy on smithi152 2024-04-02T04:55:58.872 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:58 smithi154 ceph-mon[35729]: mds.? [v2:172.21.15.154:6824/1663663935,v1:172.21.15.154:6825/1663663935] up:boot 2024-04-02T04:55:58.873 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:58 smithi154 ceph-mon[35729]: fsmap cephfs:1 {0=cephfs.smithi152.lluqrs=up:active} 1 up:standby 2024-04-02T04:55:58.873 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:58 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mds metadata", "who": "cephfs.smithi154.llvaoh"}]: dispatch 2024-04-02T04:55:58.873 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:58 smithi154 ceph-mon[35729]: fsmap cephfs:1 {0=cephfs.smithi152.lluqrs=up:active} 1 up:standby 2024-04-02T04:55:58.873 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:58 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/2083062474' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "2"}]: dispatch 2024-04-02T04:55:59.344 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-02T04:55:59.354 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi152.front.sepia.ceph.com 2024-04-02T04:55:59.354 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- bash -c 'ceph fs set cephfs allow_standby_replay true' 2024-04-02T04:55:59.596 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:59 smithi152 ceph-mon[29073]: pgmap v177: 65 pgs: 65 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 406 B/s wr, 3 op/s 2024-04-02T04:55:59.596 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:59 smithi152 ceph-mon[29073]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-02T04:55:59.596 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:59 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/2083062474' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "2"}]': finished 2024-04-02T04:55:59.596 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:59 smithi152 ceph-mon[29073]: daemon mds.cephfs.smithi154.llvaoh assigned to filesystem cephfs as rank 1 (now has 2 ranks) 2024-04-02T04:55:59.596 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:59 smithi152 ceph-mon[29073]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-02T04:55:59.596 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:59 smithi152 ceph-mon[29073]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-02T04:55:59.596 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:59 smithi152 ceph-mon[29073]: fsmap cephfs:1 {0=cephfs.smithi152.lluqrs=up:active} 1 up:standby 2024-04-02T04:55:59.596 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:59 smithi152 ceph-mon[29073]: fsmap cephfs:2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi154.llvaoh=up:creating} 2024-04-02T04:55:59.596 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:55:59 smithi152 ceph-mon[29073]: daemon mds.cephfs.smithi154.llvaoh is now active in filesystem cephfs as rank 1 2024-04-02T04:55:59.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:59 smithi154 ceph-mon[35729]: pgmap v177: 65 pgs: 65 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 406 B/s wr, 3 op/s 2024-04-02T04:55:59.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:59 smithi154 ceph-mon[35729]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-02T04:55:59.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:59 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/2083062474' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "max_mds", "val": "2"}]': finished 2024-04-02T04:55:59.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:59 smithi154 ceph-mon[35729]: daemon mds.cephfs.smithi154.llvaoh assigned to filesystem cephfs as rank 1 (now has 2 ranks) 2024-04-02T04:55:59.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:59 smithi154 ceph-mon[35729]: Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-02T04:55:59.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:59 smithi154 ceph-mon[35729]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-02T04:55:59.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:59 smithi154 ceph-mon[35729]: fsmap cephfs:1 {0=cephfs.smithi152.lluqrs=up:active} 1 up:standby 2024-04-02T04:55:59.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:59 smithi154 ceph-mon[35729]: fsmap cephfs:2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi154.llvaoh=up:creating} 2024-04-02T04:55:59.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:55:59 smithi154 ceph-mon[35729]: daemon mds.cephfs.smithi154.llvaoh is now active in filesystem cephfs as rank 1 2024-04-02T04:56:00.111 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:56:00.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:00 smithi152 ceph-mon[29073]: mds.? [v2:172.21.15.154:6824/1663663935,v1:172.21.15.154:6825/1663663935] up:active 2024-04-02T04:56:00.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:00 smithi152 ceph-mon[29073]: fsmap cephfs:2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi154.llvaoh=up:active} 2024-04-02T04:56:00.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:00 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:00.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:00 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:00.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:00 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:00.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:00 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get-or-create", "entity": "mds.cephfs.smithi154.rbkmwp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-02T04:56:00.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:00 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd='[{"prefix": "auth get-or-create", "entity": "mds.cephfs.smithi154.rbkmwp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-02T04:56:00.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:00 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:56:00.788 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:00 smithi154 ceph-mon[35729]: mds.? [v2:172.21.15.154:6824/1663663935,v1:172.21.15.154:6825/1663663935] up:active 2024-04-02T04:56:00.788 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:00 smithi154 ceph-mon[35729]: fsmap cephfs:2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi154.llvaoh=up:active} 2024-04-02T04:56:00.788 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:00 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:00.788 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:00 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:00.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:00 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:00.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:00 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get-or-create", "entity": "mds.cephfs.smithi154.rbkmwp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-02T04:56:00.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:00 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd='[{"prefix": "auth get-or-create", "entity": "mds.cephfs.smithi154.rbkmwp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-02T04:56:00.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:00 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:56:01.597 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.593+0000 7f73a64e2700 1 -- 172.21.15.152:0/2864282081 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f73a00fffb0 msgr2=0x7f73a0102390 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:01.597 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.593+0000 7f73a64e2700 1 --2- 172.21.15.152:0/2864282081 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f73a00fffb0 0x7f73a0102390 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 crypto rx=0x7f7390004940 tx=0x7f7390009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:01.598 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.593+0000 7f73a64e2700 1 -- 172.21.15.152:0/2864282081 shutdown_connections 2024-04-02T04:56:01.598 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.593+0000 7f73a64e2700 1 --2- 172.21.15.152:0/2864282081 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f73a00fffb0 0x7f73a0102390 unknown :-1 s=CLOSED pgs=242 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:01.598 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.593+0000 7f73a64e2700 1 --2- 172.21.15.152:0/2864282081 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f73a0066f10 0x7f73a00ffa70 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:01.598 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.593+0000 7f73a64e2700 1 -- 172.21.15.152:0/2864282081 >> 172.21.15.152:0/2864282081 conn(0x7f73a00f7480 msgr2=0x7f73a00f9890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:01.598 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.593+0000 7f73a64e2700 1 -- 172.21.15.152:0/2864282081 shutdown_connections 2024-04-02T04:56:01.598 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.593+0000 7f73a64e2700 1 -- 172.21.15.152:0/2864282081 wait complete. 2024-04-02T04:56:01.598 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.594+0000 7f73a64e2700 1 Processor -- start 2024-04-02T04:56:01.598 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.594+0000 7f73a64e2700 1 -- start start 2024-04-02T04:56:01.599 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.594+0000 7f73a64e2700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f73a0066f10 0x7f73a00fd2c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:01.600 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.594+0000 7f73a64e2700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f73a00fffb0 0x7f73a00fb970 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:01.600 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.594+0000 7f73a64e2700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f73a00fd8c0 con 0x7f73a00fffb0 2024-04-02T04:56:01.600 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.594+0000 7f73a64e2700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f73a00fbeb0 con 0x7f73a0066f10 2024-04-02T04:56:01.600 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.594+0000 7f739f7fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f73a00fffb0 0x7f73a00fb970 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:01.600 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.594+0000 7f739f7fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f73a00fffb0 0x7f73a00fb970 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:33918/0 (socket says 172.21.15.152:33918) 2024-04-02T04:56:01.600 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.594+0000 7f739f7fe700 1 -- 172.21.15.152:0/3396846916 learned_addr learned my addr 172.21.15.152:0/3396846916 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:56:01.600 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.594+0000 7f739ffff700 1 --2- 172.21.15.152:0/3396846916 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f73a0066f10 0x7f73a00fd2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:01.600 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.594+0000 7f739f7fe700 1 -- 172.21.15.152:0/3396846916 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f73a0066f10 msgr2=0x7f73a00fd2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:01.600 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.594+0000 7f739f7fe700 1 --2- 172.21.15.152:0/3396846916 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f73a0066f10 0x7f73a00fd2c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:01.600 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.595+0000 7f739f7fe700 1 -- 172.21.15.152:0/3396846916 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7390005040 con 0x7f73a00fffb0 2024-04-02T04:56:01.600 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.595+0000 7f739f7fe700 1 --2- 172.21.15.152:0/3396846916 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f73a00fffb0 0x7f73a00fb970 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7f7390013870 tx=0x7f7390009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:01.600 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.595+0000 7f739cff9700 1 -- 172.21.15.152:0/3396846916 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f7390021070 con 0x7f73a00fffb0 2024-04-02T04:56:01.600 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.595+0000 7f739cff9700 1 -- 172.21.15.152:0/3396846916 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f7390009430 con 0x7f73a00fffb0 2024-04-02T04:56:01.600 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.595+0000 7f73a64e2700 1 -- 172.21.15.152:0/3396846916 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f73a00fc100 con 0x7f73a00fffb0 2024-04-02T04:56:01.601 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.595+0000 7f739cff9700 1 -- 172.21.15.152:0/3396846916 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f739000e070 con 0x7f73a00fffb0 2024-04-02T04:56:01.601 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.595+0000 7f73a64e2700 1 -- 172.21.15.152:0/3396846916 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f73a00fc640 con 0x7f73a00fffb0 2024-04-02T04:56:01.601 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.596+0000 7f73a64e2700 1 -- 172.21.15.152:0/3396846916 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f73a004c710 con 0x7f73a00fffb0 2024-04-02T04:56:01.601 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.596+0000 7f739cff9700 1 -- 172.21.15.152:0/3396846916 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f739001c750 con 0x7f73a00fffb0 2024-04-02T04:56:01.601 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.597+0000 7f739cff9700 1 --2- 172.21.15.152:0/3396846916 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f738c069240 0x7f738c06b6f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:01.601 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.597+0000 7f739cff9700 1 -- 172.21.15.152:0/3396846916 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(45..45 src has 1..45) v4 ==== 5464+0+0 (secure 0 0 0) 0x7f7390088450 con 0x7f73a00fffb0 2024-04-02T04:56:01.601 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.597+0000 7f739ffff700 1 --2- 172.21.15.152:0/3396846916 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f738c069240 0x7f738c06b6f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:01.602 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.597+0000 7f739ffff700 1 --2- 172.21.15.152:0/3396846916 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f738c069240 0x7f738c06b6f0 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f7388009b80 tx=0x7f738800b040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:01.605 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.601+0000 7f739cff9700 1 -- 172.21.15.152:0/3396846916 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f7390058210 con 0x7f73a00fffb0 2024-04-02T04:56:01.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:01 smithi152 ceph-mon[29073]: pgmap v178: 65 pgs: 65 active+clean; 579 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.5 KiB/s wr, 5 op/s 2024-04-02T04:56:01.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:01 smithi152 ceph-mon[29073]: Deploying daemon mds.cephfs.smithi154.rbkmwp on smithi154 2024-04-02T04:56:01.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:01 smithi152 ceph-mon[29073]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-02T04:56:01.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:01 smithi152 ceph-mon[29073]: Cluster is now healthy 2024-04-02T04:56:01.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:01 smithi152 ceph-mon[29073]: mds.? [v2:172.21.15.152:6828/3554217123,v1:172.21.15.152:6829/3554217123] up:boot 2024-04-02T04:56:01.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:01 smithi152 ceph-mon[29073]: fsmap cephfs:2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi154.llvaoh=up:active} 1 up:standby 2024-04-02T04:56:01.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:01 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mds metadata", "who": "cephfs.smithi152.ywghzy"}]: dispatch 2024-04-02T04:56:01.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:01 smithi154 ceph-mon[35729]: pgmap v178: 65 pgs: 65 active+clean; 579 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.5 KiB/s wr, 5 op/s 2024-04-02T04:56:01.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:01 smithi154 ceph-mon[35729]: Deploying daemon mds.cephfs.smithi154.rbkmwp on smithi154 2024-04-02T04:56:01.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:01 smithi154 ceph-mon[35729]: Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-02T04:56:01.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:01 smithi154 ceph-mon[35729]: Cluster is now healthy 2024-04-02T04:56:01.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:01 smithi154 ceph-mon[35729]: mds.? [v2:172.21.15.152:6828/3554217123,v1:172.21.15.152:6829/3554217123] up:boot 2024-04-02T04:56:01.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:01 smithi154 ceph-mon[35729]: fsmap cephfs:2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi154.llvaoh=up:active} 1 up:standby 2024-04-02T04:56:01.789 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:01 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mds metadata", "who": "cephfs.smithi152.ywghzy"}]: dispatch 2024-04-02T04:56:01.837 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:01.833+0000 7f73a64e2700 1 -- 172.21.15.152:0/3396846916 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"} v 0) v1 -- 0x7f73a0105900 con 0x7f73a00fffb0 2024-04-02T04:56:02.518 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:02.515+0000 7f739cff9700 1 -- 172.21.15.152:0/3396846916 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]=0 v12) v1 ==== 121+0+0 (secure 0 0 0) 0x7f7390004490 con 0x7f73a00fffb0 2024-04-02T04:56:02.521 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:02.517+0000 7f73a64e2700 1 -- 172.21.15.152:0/3396846916 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f738c069240 msgr2=0x7f738c06b6f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:02.521 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:02.517+0000 7f73a64e2700 1 --2- 172.21.15.152:0/3396846916 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f738c069240 0x7f738c06b6f0 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 crypto rx=0x7f7388009b80 tx=0x7f738800b040 comp rx=0 tx=0).stop 2024-04-02T04:56:02.521 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:02.517+0000 7f73a64e2700 1 -- 172.21.15.152:0/3396846916 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f73a00fffb0 msgr2=0x7f73a00fb970 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:02.521 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:02.517+0000 7f73a64e2700 1 --2- 172.21.15.152:0/3396846916 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f73a00fffb0 0x7f73a00fb970 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 crypto rx=0x7f7390013870 tx=0x7f7390009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:02.521 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:02.517+0000 7f73a64e2700 1 -- 172.21.15.152:0/3396846916 shutdown_connections 2024-04-02T04:56:02.521 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:02.517+0000 7f73a64e2700 1 --2- 172.21.15.152:0/3396846916 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f738c069240 0x7f738c06b6f0 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:02.521 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:02.517+0000 7f73a64e2700 1 --2- 172.21.15.152:0/3396846916 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f73a0066f10 0x7f73a00fd2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:02.521 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:02.517+0000 7f73a64e2700 1 --2- 172.21.15.152:0/3396846916 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f73a00fffb0 0x7f73a00fb970 unknown :-1 s=CLOSED pgs=243 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:02.521 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:02.517+0000 7f73a64e2700 1 -- 172.21.15.152:0/3396846916 >> 172.21.15.152:0/3396846916 conn(0x7f73a00f7480 msgr2=0x7f73a00f7850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:02.521 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:02.517+0000 7f73a64e2700 1 -- 172.21.15.152:0/3396846916 shutdown_connections 2024-04-02T04:56:02.522 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:02.517+0000 7f73a64e2700 1 -- 172.21.15.152:0/3396846916 wait complete. 2024-04-02T04:56:02.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:02 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/3396846916' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]: dispatch 2024-04-02T04:56:02.881 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:02 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/3396846916' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]: dispatch 2024-04-02T04:56:03.246 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-02T04:56:03.259 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi152.front.sepia.ceph.com 2024-04-02T04:56:03.259 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- bash -c 'ceph fs set cephfs inline_data true --yes-i-really-really-mean-it' 2024-04-02T04:56:03.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:03 smithi154 ceph-mon[35729]: pgmap v179: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s wr, 5 op/s 2024-04-02T04:56:03.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:03 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/3396846916' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]': finished 2024-04-02T04:56:03.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:03 smithi154 ceph-mon[35729]: mds.? [v2:172.21.15.154:6826/53707828,v1:172.21.15.154:6827/53707828] up:boot 2024-04-02T04:56:03.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:03 smithi154 ceph-mon[35729]: fsmap cephfs:2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi154.llvaoh=up:active} 2 up:standby 2024-04-02T04:56:03.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:03 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mds metadata", "who": "cephfs.smithi154.rbkmwp"}]: dispatch 2024-04-02T04:56:03.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:03 smithi154 ceph-mon[35729]: fsmap cephfs:2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi154.llvaoh=up:active} 2 up:standby-replay 2024-04-02T04:56:03.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:03 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:03.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:03 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:03.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:03 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:03.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:03 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:03.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:03 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:03.735 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:03 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:56:04.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:03 smithi152 ceph-mon[29073]: pgmap v179: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s wr, 5 op/s 2024-04-02T04:56:04.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:03 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/3396846916' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "true"}]': finished 2024-04-02T04:56:04.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:03 smithi152 ceph-mon[29073]: mds.? [v2:172.21.15.154:6826/53707828,v1:172.21.15.154:6827/53707828] up:boot 2024-04-02T04:56:04.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:03 smithi152 ceph-mon[29073]: fsmap cephfs:2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi154.llvaoh=up:active} 2 up:standby 2024-04-02T04:56:04.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:03 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mds metadata", "who": "cephfs.smithi154.rbkmwp"}]: dispatch 2024-04-02T04:56:04.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:03 smithi152 ceph-mon[29073]: fsmap cephfs:2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi154.llvaoh=up:active} 2 up:standby-replay 2024-04-02T04:56:04.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:03 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:04.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:03 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:04.008 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:03 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:04.008 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:03 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:04.008 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:03 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:04.008 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:03 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:56:04.071 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:56:04.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:04 smithi152 ceph-mon[29073]: pgmap v180: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.0 KiB/s rd, 2.1 KiB/s wr, 8 op/s 2024-04-02T04:56:04.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:04 smithi152 ceph-mon[29073]: mds.? [v2:172.21.15.152:6828/3554217123,v1:172.21.15.152:6829/3554217123] up:standby-replay 2024-04-02T04:56:04.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:04 smithi152 ceph-mon[29073]: Dropping low affinity active daemon mds.cephfs.smithi154.llvaoh in favor of higher affinity standby. 2024-04-02T04:56:04.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:04 smithi152 ceph-mon[29073]: Replacing daemon mds.cephfs.smithi154.llvaoh as rank 1 with standby daemon mds.cephfs.smithi152.ywghzy 2024-04-02T04:56:04.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:04 smithi152 ceph-mon[29073]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-02T04:56:04.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:04 smithi152 ceph-mon[29073]: fsmap cephfs:2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi154.llvaoh=up:active} 2 up:standby-replay 2024-04-02T04:56:04.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:04 smithi152 ceph-mon[29073]: osdmap e46: 6 total, 6 up, 6 in 2024-04-02T04:56:04.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:04 smithi152 ceph-mon[29073]: fsmap cephfs:2/2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi152.ywghzy=up:replay} 1 up:standby-replay 2024-04-02T04:56:04.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:04 smithi154 ceph-mon[35729]: pgmap v180: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.0 KiB/s rd, 2.1 KiB/s wr, 8 op/s 2024-04-02T04:56:04.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:04 smithi154 ceph-mon[35729]: mds.? [v2:172.21.15.152:6828/3554217123,v1:172.21.15.152:6829/3554217123] up:standby-replay 2024-04-02T04:56:04.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:04 smithi154 ceph-mon[35729]: Dropping low affinity active daemon mds.cephfs.smithi154.llvaoh in favor of higher affinity standby. 2024-04-02T04:56:04.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:04 smithi154 ceph-mon[35729]: Replacing daemon mds.cephfs.smithi154.llvaoh as rank 1 with standby daemon mds.cephfs.smithi152.ywghzy 2024-04-02T04:56:04.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:04 smithi154 ceph-mon[35729]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-02T04:56:04.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:04 smithi154 ceph-mon[35729]: fsmap cephfs:2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi154.llvaoh=up:active} 2 up:standby-replay 2024-04-02T04:56:04.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:04 smithi154 ceph-mon[35729]: osdmap e46: 6 total, 6 up, 6 in 2024-04-02T04:56:04.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:04 smithi154 ceph-mon[35729]: fsmap cephfs:2/2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi152.ywghzy=up:replay} 1 up:standby-replay 2024-04-02T04:56:05.855 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:05 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:05.855 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:05 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:05.855 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:05 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:05.855 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:05 smithi152 ceph-mon[29073]: mds.? [v2:172.21.15.152:6828/3554217123,v1:172.21.15.152:6829/3554217123] up:resolve 2024-04-02T04:56:05.855 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:05 smithi152 ceph-mon[29073]: mds.? [v2:172.21.15.154:6824/1569612467,v1:172.21.15.154:6825/1569612467] up:boot 2024-04-02T04:56:05.855 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:05 smithi152 ceph-mon[29073]: fsmap cephfs:2/2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi152.ywghzy=up:resolve} 1 up:standby-replay 1 up:standby 2024-04-02T04:56:05.855 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:05 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mds metadata", "who": "cephfs.smithi154.llvaoh"}]: dispatch 2024-04-02T04:56:05.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:05 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:05.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:05 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:05.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:05 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:05.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:05 smithi154 ceph-mon[35729]: mds.? [v2:172.21.15.152:6828/3554217123,v1:172.21.15.152:6829/3554217123] up:resolve 2024-04-02T04:56:05.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:05 smithi154 ceph-mon[35729]: mds.? [v2:172.21.15.154:6824/1569612467,v1:172.21.15.154:6825/1569612467] up:boot 2024-04-02T04:56:05.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:05 smithi154 ceph-mon[35729]: fsmap cephfs:2/2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi152.ywghzy=up:resolve} 1 up:standby-replay 1 up:standby 2024-04-02T04:56:05.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:05 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mds metadata", "who": "cephfs.smithi154.llvaoh"}]: dispatch 2024-04-02T04:56:06.686 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.569+0000 7f2fa6c0c700 1 -- 172.21.15.152:0/3048088757 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2fa00fdfe0 msgr2=0x7f2fa00fe450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:06.686 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.569+0000 7f2fa6c0c700 1 --2- 172.21.15.152:0/3048088757 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2fa00fdfe0 0x7f2fa00fe450 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 crypto rx=0x7f2f8c004940 tx=0x7f2f8c009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:06.686 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.569+0000 7f2fa6c0c700 1 -- 172.21.15.152:0/3048088757 shutdown_connections 2024-04-02T04:56:06.686 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.569+0000 7f2fa6c0c700 1 --2- 172.21.15.152:0/3048088757 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2fa00fdfe0 0x7f2fa00fe450 unknown :-1 s=CLOSED pgs=244 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:06.686 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.569+0000 7f2fa6c0c700 1 --2- 172.21.15.152:0/3048088757 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2fa00fce70 0x7f2fa00fd280 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:06.686 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.569+0000 7f2fa6c0c700 1 -- 172.21.15.152:0/3048088757 >> 172.21.15.152:0/3048088757 conn(0x7f2fa00f8550 msgr2=0x7f2fa00fa960 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:06.687 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.569+0000 7f2fa6c0c700 1 -- 172.21.15.152:0/3048088757 shutdown_connections 2024-04-02T04:56:06.687 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.569+0000 7f2fa6c0c700 1 -- 172.21.15.152:0/3048088757 wait complete. 2024-04-02T04:56:06.687 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.570+0000 7f2fa6c0c700 1 Processor -- start 2024-04-02T04:56:06.687 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.570+0000 7f2fa6c0c700 1 -- start start 2024-04-02T04:56:06.687 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.570+0000 7f2fa6c0c700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2fa00fce70 0x7f2fa010d8c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:06.687 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.570+0000 7f2fa6c0c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2fa00fdfe0 0x7f2fa010de00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:06.687 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.570+0000 7f2fa6c0c700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2fa010e400 con 0x7f2fa00fdfe0 2024-04-02T04:56:06.687 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.570+0000 7f2fa6c0c700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2fa010e570 con 0x7f2fa00fce70 2024-04-02T04:56:06.687 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.570+0000 7f2f9ffff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2fa00fdfe0 0x7f2fa010de00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:06.687 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.570+0000 7f2f9ffff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2fa00fdfe0 0x7f2fa010de00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:33936/0 (socket says 172.21.15.152:33936) 2024-04-02T04:56:06.687 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.570+0000 7f2f9ffff700 1 -- 172.21.15.152:0/4247644291 learned_addr learned my addr 172.21.15.152:0/4247644291 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:56:06.687 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.570+0000 7f2fa49a8700 1 --2- 172.21.15.152:0/4247644291 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2fa00fce70 0x7f2fa010d8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:06.688 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.570+0000 7f2f9ffff700 1 -- 172.21.15.152:0/4247644291 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2fa00fce70 msgr2=0x7f2fa010d8c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:06.688 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.570+0000 7f2f9ffff700 1 --2- 172.21.15.152:0/4247644291 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2fa00fce70 0x7f2fa010d8c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:06.688 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.570+0000 7f2f9ffff700 1 -- 172.21.15.152:0/4247644291 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2f8c005040 con 0x7f2fa00fdfe0 2024-04-02T04:56:06.688 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.571+0000 7f2f9ffff700 1 --2- 172.21.15.152:0/4247644291 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2fa00fdfe0 0x7f2fa010de00 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7f2f8c009150 tx=0x7f2f8c009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:06.688 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.571+0000 7f2f9d7fa700 1 -- 172.21.15.152:0/4247644291 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f2f8c00e040 con 0x7f2fa00fdfe0 2024-04-02T04:56:06.688 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.571+0000 7f2fa6c0c700 1 -- 172.21.15.152:0/4247644291 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2fa0112f70 con 0x7f2fa00fdfe0 2024-04-02T04:56:06.688 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.571+0000 7f2f9d7fa700 1 -- 172.21.15.152:0/4247644291 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f2f8c004290 con 0x7f2fa00fdfe0 2024-04-02T04:56:06.688 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.571+0000 7f2f9d7fa700 1 -- 172.21.15.152:0/4247644291 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f2f8c01c630 con 0x7f2fa00fdfe0 2024-04-02T04:56:06.688 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.571+0000 7f2fa6c0c700 1 -- 172.21.15.152:0/4247644291 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2fa0113500 con 0x7f2fa00fdfe0 2024-04-02T04:56:06.688 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.572+0000 7f2fa6c0c700 1 -- 172.21.15.152:0/4247644291 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2fa004c710 con 0x7f2fa00fdfe0 2024-04-02T04:56:06.688 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.572+0000 7f2f9d7fa700 1 -- 172.21.15.152:0/4247644291 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f2f8c00bbf0 con 0x7f2fa00fdfe0 2024-04-02T04:56:06.688 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.572+0000 7f2f9d7fa700 1 --2- 172.21.15.152:0/4247644291 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2f900692a0 0x7f2f9006b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:06.689 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.573+0000 7f2f9d7fa700 1 -- 172.21.15.152:0/4247644291 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7f2f8c087e80 con 0x7f2fa00fdfe0 2024-04-02T04:56:06.689 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.573+0000 7f2fa49a8700 1 --2- 172.21.15.152:0/4247644291 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2f900692a0 0x7f2f9006b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:06.689 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.573+0000 7f2fa49a8700 1 --2- 172.21.15.152:0/4247644291 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2f900692a0 0x7f2f9006b750 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f2f94009fd0 tx=0x7f2f9400b040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:06.689 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.577+0000 7f2f9d7fa700 1 -- 172.21.15.152:0/4247644291 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f2f8c057bf0 con 0x7f2fa00fdfe0 2024-04-02T04:56:06.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:06 smithi152 ceph-mon[29073]: pgmap v182: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.2 KiB/s rd, 2.4 KiB/s wr, 8 op/s 2024-04-02T04:56:06.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:06 smithi152 ceph-mon[29073]: mds.? [v2:172.21.15.152:6828/3554217123,v1:172.21.15.152:6829/3554217123] up:reconnect 2024-04-02T04:56:06.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:06 smithi152 ceph-mon[29073]: mds.? [v2:172.21.15.152:6826/4038990242,v1:172.21.15.152:6827/4038990242] up:active 2024-04-02T04:56:06.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:06 smithi152 ceph-mon[29073]: mds.? [v2:172.21.15.154:6826/53707828,v1:172.21.15.154:6827/53707828] up:standby-replay 2024-04-02T04:56:06.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:06 smithi152 ceph-mon[29073]: fsmap cephfs:2/2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi152.ywghzy=up:reconnect} 1 up:standby-replay 1 up:standby 2024-04-02T04:56:06.817 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:06.813+0000 7f2fa6c0c700 1 -- 172.21.15.152:0/4247644291 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "fs set", "fs_name": "cephfs", "var": "inline_data", "val": "true", "yes_i_really_really_mean_it": true} v 0) v1 -- 0x7f2fa0113720 con 0x7f2fa00fdfe0 2024-04-02T04:56:06.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:06 smithi154 ceph-mon[35729]: pgmap v182: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.2 KiB/s rd, 2.4 KiB/s wr, 8 op/s 2024-04-02T04:56:06.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:06 smithi154 ceph-mon[35729]: mds.? [v2:172.21.15.152:6828/3554217123,v1:172.21.15.152:6829/3554217123] up:reconnect 2024-04-02T04:56:06.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:06 smithi154 ceph-mon[35729]: mds.? [v2:172.21.15.152:6826/4038990242,v1:172.21.15.152:6827/4038990242] up:active 2024-04-02T04:56:06.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:06 smithi154 ceph-mon[35729]: mds.? [v2:172.21.15.154:6826/53707828,v1:172.21.15.154:6827/53707828] up:standby-replay 2024-04-02T04:56:06.984 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:06 smithi154 ceph-mon[35729]: fsmap cephfs:2/2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi152.ywghzy=up:reconnect} 1 up:standby-replay 1 up:standby 2024-04-02T04:56:07.535 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:07.531+0000 7f2f9d7fa700 1 -- 172.21.15.152:0/4247644291 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "fs set", "fs_name": "cephfs", "var": "inline_data", "val": "true", "yes_i_really_really_mean_it": true}]=0 inline data enabled v18) v1 ==== 168+0+0 (secure 0 0 0) 0x7f2f8c057780 con 0x7f2fa00fdfe0 2024-04-02T04:56:07.537 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:07.533+0000 7f2fa6c0c700 1 -- 172.21.15.152:0/4247644291 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2f900692a0 msgr2=0x7f2f9006b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:07.537 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:07.533+0000 7f2fa6c0c700 1 --2- 172.21.15.152:0/4247644291 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2f900692a0 0x7f2f9006b750 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 crypto rx=0x7f2f94009fd0 tx=0x7f2f9400b040 comp rx=0 tx=0).stop 2024-04-02T04:56:07.537 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:07.533+0000 7f2fa6c0c700 1 -- 172.21.15.152:0/4247644291 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2fa00fdfe0 msgr2=0x7f2fa010de00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:07.537 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:07.533+0000 7f2fa6c0c700 1 --2- 172.21.15.152:0/4247644291 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2fa00fdfe0 0x7f2fa010de00 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 crypto rx=0x7f2f8c009150 tx=0x7f2f8c009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:07.537 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:07.533+0000 7f2fa6c0c700 1 -- 172.21.15.152:0/4247644291 shutdown_connections 2024-04-02T04:56:07.537 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:07.533+0000 7f2fa6c0c700 1 --2- 172.21.15.152:0/4247644291 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2f900692a0 0x7f2f9006b750 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:07.537 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:07.533+0000 7f2fa6c0c700 1 --2- 172.21.15.152:0/4247644291 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2fa00fce70 0x7f2fa010d8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:07.537 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:07.533+0000 7f2fa6c0c700 1 --2- 172.21.15.152:0/4247644291 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2fa00fdfe0 0x7f2fa010de00 unknown :-1 s=CLOSED pgs=245 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:07.537 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:07.533+0000 7f2fa6c0c700 1 -- 172.21.15.152:0/4247644291 >> 172.21.15.152:0/4247644291 conn(0x7f2fa00f8550 msgr2=0x7f2fa00f9bc0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:07.537 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:07.533+0000 7f2fa6c0c700 1 -- 172.21.15.152:0/4247644291 shutdown_connections 2024-04-02T04:56:07.538 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:07.533+0000 7f2fa6c0c700 1 -- 172.21.15.152:0/4247644291 wait complete. 2024-04-02T04:56:07.538 INFO:teuthology.orchestra.run.smithi152.stderr:inline data enabled 2024-04-02T04:56:07.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:07 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/4247644291' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "inline_data", "val": "true", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-02T04:56:08.092 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:07 smithi154 ceph-mon[35729]: Health check failed: 1 filesystem with deprecated feature inline_data (FS_INLINE_DATA_DEPRECATED) 2024-04-02T04:56:08.092 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:07 smithi154 ceph-mon[35729]: mds.? [v2:172.21.15.152:6828/3554217123,v1:172.21.15.152:6829/3554217123] up:rejoin 2024-04-02T04:56:08.092 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:07 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/4247644291' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "inline_data", "val": "true", "yes_i_really_really_mean_it": true}]': finished 2024-04-02T04:56:08.092 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:07 smithi154 ceph-mon[35729]: fsmap cephfs:2/2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi152.ywghzy=up:rejoin} 1 up:standby-replay 1 up:standby 2024-04-02T04:56:08.092 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:07 smithi154 ceph-mon[35729]: daemon mds.cephfs.smithi152.ywghzy is now active in filesystem cephfs as rank 1 2024-04-02T04:56:08.093 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:07 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/4247644291' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "inline_data", "val": "true", "yes_i_really_really_mean_it": true}]: dispatch 2024-04-02T04:56:08.093 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:07 smithi152 ceph-mon[29073]: Health check failed: 1 filesystem with deprecated feature inline_data (FS_INLINE_DATA_DEPRECATED) 2024-04-02T04:56:08.093 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:07 smithi152 ceph-mon[29073]: mds.? [v2:172.21.15.152:6828/3554217123,v1:172.21.15.152:6829/3554217123] up:rejoin 2024-04-02T04:56:08.093 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:07 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/4247644291' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "inline_data", "val": "true", "yes_i_really_really_mean_it": true}]': finished 2024-04-02T04:56:08.093 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:07 smithi152 ceph-mon[29073]: fsmap cephfs:2/2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi152.ywghzy=up:rejoin} 1 up:standby-replay 1 up:standby 2024-04-02T04:56:08.093 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:07 smithi152 ceph-mon[29073]: daemon mds.cephfs.smithi152.ywghzy is now active in filesystem cephfs as rank 1 2024-04-02T04:56:08.622 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-02T04:56:08.632 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi152.front.sepia.ceph.com 2024-04-02T04:56:08.633 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- bash -c 'ceph fs dump' 2024-04-02T04:56:08.899 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:08 smithi152 ceph-mon[29073]: pgmap v183: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.9 KiB/s rd, 2.1 KiB/s wr, 8 op/s 2024-04-02T04:56:08.899 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:08 smithi152 ceph-mon[29073]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-02T04:56:08.899 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:08 smithi152 ceph-mon[29073]: mds.? [v2:172.21.15.152:6828/3554217123,v1:172.21.15.152:6829/3554217123] up:active 2024-04-02T04:56:08.899 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:08 smithi152 ceph-mon[29073]: fsmap cephfs:2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi152.ywghzy=up:active} 1 up:standby-replay 1 up:standby 2024-04-02T04:56:08.899 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:08 smithi152 ceph-mon[29073]: fsmap cephfs:2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi152.ywghzy=up:active} 2 up:standby-replay 2024-04-02T04:56:08.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:08 smithi154 ceph-mon[35729]: pgmap v183: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.9 KiB/s rd, 2.1 KiB/s wr, 8 op/s 2024-04-02T04:56:08.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:08 smithi154 ceph-mon[35729]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-02T04:56:08.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:08 smithi154 ceph-mon[35729]: mds.? [v2:172.21.15.152:6828/3554217123,v1:172.21.15.152:6829/3554217123] up:active 2024-04-02T04:56:08.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:08 smithi154 ceph-mon[35729]: fsmap cephfs:2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi152.ywghzy=up:active} 1 up:standby-replay 1 up:standby 2024-04-02T04:56:08.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:08 smithi154 ceph-mon[35729]: fsmap cephfs:2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi152.ywghzy=up:active} 2 up:standby-replay 2024-04-02T04:56:09.467 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:56:11.170 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.164+0000 7f84449f3700 1 -- 172.21.15.152:0/2610068534 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f84400fce70 msgr2=0x7f84400fd280 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:11.171 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.164+0000 7f84449f3700 1 --2- 172.21.15.152:0/2610068534 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f84400fce70 0x7f84400fd280 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 crypto rx=0x7f8430004940 tx=0x7f8430009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:11.171 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.164+0000 7f84449f3700 1 -- 172.21.15.152:0/2610068534 shutdown_connections 2024-04-02T04:56:11.171 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.164+0000 7f84449f3700 1 --2- 172.21.15.152:0/2610068534 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f84400fdfe0 0x7f84400fe450 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:11.171 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.164+0000 7f84449f3700 1 --2- 172.21.15.152:0/2610068534 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f84400fce70 0x7f84400fd280 unknown :-1 s=CLOSED pgs=246 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:11.171 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.164+0000 7f84449f3700 1 -- 172.21.15.152:0/2610068534 >> 172.21.15.152:0/2610068534 conn(0x7f84400f8550 msgr2=0x7f84400fa960 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:11.171 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.164+0000 7f84449f3700 1 -- 172.21.15.152:0/2610068534 shutdown_connections 2024-04-02T04:56:11.171 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.164+0000 7f84449f3700 1 -- 172.21.15.152:0/2610068534 wait complete. 2024-04-02T04:56:11.171 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.165+0000 7f84449f3700 1 Processor -- start 2024-04-02T04:56:11.171 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.165+0000 7f84449f3700 1 -- start start 2024-04-02T04:56:11.171 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.165+0000 7f84449f3700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f84400fce70 0x7f844010fb00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:11.171 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.165+0000 7f84449f3700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f84400fdfe0 0x7f8440110040 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:11.171 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.165+0000 7f84449f3700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8440110640 con 0x7f84400fce70 2024-04-02T04:56:11.171 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.165+0000 7f84449f3700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f84401107b0 con 0x7f84400fdfe0 2024-04-02T04:56:11.172 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.165+0000 7f843e59c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f84400fce70 0x7f844010fb00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:11.172 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.165+0000 7f843e59c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f84400fce70 0x7f844010fb00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:48536/0 (socket says 172.21.15.152:48536) 2024-04-02T04:56:11.172 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.165+0000 7f843e59c700 1 -- 172.21.15.152:0/537103280 learned_addr learned my addr 172.21.15.152:0/537103280 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:56:11.172 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.165+0000 7f843dd9b700 1 --2- 172.21.15.152:0/537103280 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f84400fdfe0 0x7f8440110040 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:11.172 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.165+0000 7f843e59c700 1 -- 172.21.15.152:0/537103280 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f84400fdfe0 msgr2=0x7f8440110040 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:11.172 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.165+0000 7f843e59c700 1 --2- 172.21.15.152:0/537103280 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f84400fdfe0 0x7f8440110040 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:11.172 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.166+0000 7f843e59c700 1 -- 172.21.15.152:0/537103280 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8430005040 con 0x7f84400fce70 2024-04-02T04:56:11.172 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.166+0000 7f843e59c700 1 --2- 172.21.15.152:0/537103280 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f84400fce70 0x7f844010fb00 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7f8430013870 tx=0x7f8430009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:11.172 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.166+0000 7f842effd700 1 -- 172.21.15.152:0/537103280 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f8430021070 con 0x7f84400fce70 2024-04-02T04:56:11.172 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.166+0000 7f84449f3700 1 -- 172.21.15.152:0/537103280 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f84401151b0 con 0x7f84400fce70 2024-04-02T04:56:11.172 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.166+0000 7f842effd700 1 -- 172.21.15.152:0/537103280 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f8430009430 con 0x7f84400fce70 2024-04-02T04:56:11.172 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.166+0000 7f842effd700 1 -- 172.21.15.152:0/537103280 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f843000e070 con 0x7f84400fce70 2024-04-02T04:56:11.172 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.166+0000 7f84449f3700 1 -- 172.21.15.152:0/537103280 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f84401156f0 con 0x7f84400fce70 2024-04-02T04:56:11.172 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.167+0000 7f84449f3700 1 -- 172.21.15.152:0/537103280 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8440101970 con 0x7f84400fce70 2024-04-02T04:56:11.173 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.167+0000 7f842effd700 1 -- 172.21.15.152:0/537103280 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f843001c750 con 0x7f84400fce70 2024-04-02T04:56:11.173 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.168+0000 7f842effd700 1 --2- 172.21.15.152:0/537103280 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f8428069240 0x7f842806b6f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:11.173 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.168+0000 7f842effd700 1 -- 172.21.15.152:0/537103280 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7f84300884a0 con 0x7f84400fce70 2024-04-02T04:56:11.173 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.168+0000 7f843dd9b700 1 --2- 172.21.15.152:0/537103280 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f8428069240 0x7f842806b6f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:11.173 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.168+0000 7f843dd9b700 1 --2- 172.21.15.152:0/537103280 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f8428069240 0x7f842806b6f0 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f8434009b80 tx=0x7f843400b040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:11.176 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.172+0000 7f842effd700 1 -- 172.21.15.152:0/537103280 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f8430058210 con 0x7f84400fce70 2024-04-02T04:56:11.176 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:11 smithi152 ceph-mon[29073]: pgmap v184: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 3.0 KiB/s rd, 1.1 KiB/s wr, 7 op/s 2024-04-02T04:56:11.176 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:11 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:11.176 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:11 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:56:11.427 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.422+0000 7f84449f3700 1 -- 172.21.15.152:0/537103280 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "fs dump"} v 0) v1 -- 0x7f844002b340 con 0x7f84400fce70 2024-04-02T04:56:11.427 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.423+0000 7f842effd700 1 -- 172.21.15.152:0/537103280 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump"}]=0 dumped fsmap epoch 20 v20) v1 ==== 76+0+1833 (secure 0 0 0) 0x7f8430057da0 con 0x7f84400fce70 2024-04-02T04:56:11.427 INFO:teuthology.orchestra.run.smithi152.stdout:e20 2024-04-02T04:56:11.427 INFO:teuthology.orchestra.run.smithi152.stdout:enable_multiple, ever_enabled_multiple: 1,1 2024-04-02T04:56:11.427 INFO:teuthology.orchestra.run.smithi152.stdout:default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-02T04:56:11.427 INFO:teuthology.orchestra.run.smithi152.stdout:legacy client fscid: 1 2024-04-02T04:56:11.427 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:56:11.428 INFO:teuthology.orchestra.run.smithi152.stdout:Filesystem 'cephfs' (1) 2024-04-02T04:56:11.428 INFO:teuthology.orchestra.run.smithi152.stdout:fs_name cephfs 2024-04-02T04:56:11.428 INFO:teuthology.orchestra.run.smithi152.stdout:epoch 20 2024-04-02T04:56:11.428 INFO:teuthology.orchestra.run.smithi152.stdout:flags 32 joinable allow_snaps allow_multimds_snaps allow_standby_replay 2024-04-02T04:56:11.428 INFO:teuthology.orchestra.run.smithi152.stdout:created 2024-04-02T04:55:50.446442+0000 2024-04-02T04:56:11.428 INFO:teuthology.orchestra.run.smithi152.stdout:modified 2024-04-02T04:56:08.533665+0000 2024-04-02T04:56:11.428 INFO:teuthology.orchestra.run.smithi152.stdout:tableserver 0 2024-04-02T04:56:11.428 INFO:teuthology.orchestra.run.smithi152.stdout:root 0 2024-04-02T04:56:11.428 INFO:teuthology.orchestra.run.smithi152.stdout:session_timeout 60 2024-04-02T04:56:11.428 INFO:teuthology.orchestra.run.smithi152.stdout:session_autoclose 300 2024-04-02T04:56:11.428 INFO:teuthology.orchestra.run.smithi152.stdout:max_file_size 1099511627776 2024-04-02T04:56:11.428 INFO:teuthology.orchestra.run.smithi152.stdout:required_client_features {} 2024-04-02T04:56:11.428 INFO:teuthology.orchestra.run.smithi152.stdout:last_failure 0 2024-04-02T04:56:11.428 INFO:teuthology.orchestra.run.smithi152.stdout:last_failure_osd_epoch 46 2024-04-02T04:56:11.429 INFO:teuthology.orchestra.run.smithi152.stdout:compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-02T04:56:11.429 INFO:teuthology.orchestra.run.smithi152.stdout:max_mds 2 2024-04-02T04:56:11.429 INFO:teuthology.orchestra.run.smithi152.stdout:in 0,1 2024-04-02T04:56:11.429 INFO:teuthology.orchestra.run.smithi152.stdout:up {0=14456,1=14470} 2024-04-02T04:56:11.429 INFO:teuthology.orchestra.run.smithi152.stdout:failed 2024-04-02T04:56:11.429 INFO:teuthology.orchestra.run.smithi152.stdout:damaged 2024-04-02T04:56:11.429 INFO:teuthology.orchestra.run.smithi152.stdout:stopped 2024-04-02T04:56:11.429 INFO:teuthology.orchestra.run.smithi152.stdout:data_pools [3] 2024-04-02T04:56:11.429 INFO:teuthology.orchestra.run.smithi152.stdout:metadata_pool 2 2024-04-02T04:56:11.429 INFO:teuthology.orchestra.run.smithi152.stdout:inline_data enabled 2024-04-02T04:56:11.429 INFO:teuthology.orchestra.run.smithi152.stdout:balancer 2024-04-02T04:56:11.429 INFO:teuthology.orchestra.run.smithi152.stdout:standby_count_wanted 1 2024-04-02T04:56:11.429 INFO:teuthology.orchestra.run.smithi152.stdout:[mds.cephfs.smithi152.lluqrs{0:14456} state up:active seq 5 join_fscid=1 addr [v2:172.21.15.152:6826/4038990242,v1:172.21.15.152:6827/4038990242] compat {c=[1],r=[1],i=[7ff]}] 2024-04-02T04:56:11.430 INFO:teuthology.orchestra.run.smithi152.stdout:[mds.cephfs.smithi154.rbkmwp{0:24297} state up:standby-replay seq 2 join_fscid=1 addr [v2:172.21.15.154:6826/53707828,v1:172.21.15.154:6827/53707828] compat {c=[1],r=[1],i=[7ff]}] 2024-04-02T04:56:11.430 INFO:teuthology.orchestra.run.smithi152.stdout:[mds.cephfs.smithi152.ywghzy{1:14470} state up:active seq 6 join_fscid=1 addr [v2:172.21.15.152:6828/3554217123,v1:172.21.15.152:6829/3554217123] compat {c=[1],r=[1],i=[7ff]}] 2024-04-02T04:56:11.430 INFO:teuthology.orchestra.run.smithi152.stdout:[mds.cephfs.smithi154.llvaoh{1:24301} state up:standby-replay seq 1 join_fscid=1 addr [v2:172.21.15.154:6824/1569612467,v1:172.21.15.154:6825/1569612467] compat {c=[1],r=[1],i=[7ff]}] 2024-04-02T04:56:11.430 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:56:11.430 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:56:11.431 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.425+0000 7f84449f3700 1 -- 172.21.15.152:0/537103280 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f8428069240 msgr2=0x7f842806b6f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:11.431 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.425+0000 7f84449f3700 1 --2- 172.21.15.152:0/537103280 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f8428069240 0x7f842806b6f0 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 crypto rx=0x7f8434009b80 tx=0x7f843400b040 comp rx=0 tx=0).stop 2024-04-02T04:56:11.431 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.425+0000 7f84449f3700 1 -- 172.21.15.152:0/537103280 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f84400fce70 msgr2=0x7f844010fb00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:11.431 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.425+0000 7f84449f3700 1 --2- 172.21.15.152:0/537103280 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f84400fce70 0x7f844010fb00 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 crypto rx=0x7f8430013870 tx=0x7f8430009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:11.431 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.425+0000 7f84449f3700 1 -- 172.21.15.152:0/537103280 shutdown_connections 2024-04-02T04:56:11.431 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.425+0000 7f84449f3700 1 --2- 172.21.15.152:0/537103280 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f8428069240 0x7f842806b6f0 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:11.431 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.425+0000 7f84449f3700 1 --2- 172.21.15.152:0/537103280 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f84400fce70 0x7f844010fb00 unknown :-1 s=CLOSED pgs=247 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:11.431 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.425+0000 7f84449f3700 1 --2- 172.21.15.152:0/537103280 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f84400fdfe0 0x7f8440110040 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:11.432 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.425+0000 7f84449f3700 1 -- 172.21.15.152:0/537103280 >> 172.21.15.152:0/537103280 conn(0x7f84400f8550 msgr2=0x7f84400f9af0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:11.432 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.425+0000 7f84449f3700 1 -- 172.21.15.152:0/537103280 shutdown_connections 2024-04-02T04:56:11.432 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:11.425+0000 7f84449f3700 1 -- 172.21.15.152:0/537103280 wait complete. 2024-04-02T04:56:11.432 INFO:teuthology.orchestra.run.smithi152.stderr:dumped fsmap epoch 20 2024-04-02T04:56:11.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:11 smithi154 ceph-mon[35729]: pgmap v184: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 3.0 KiB/s rd, 1.1 KiB/s wr, 7 op/s 2024-04-02T04:56:11.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:11 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:11.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:11 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:56:12.074 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- bash -c 'ceph --format=json fs dump | jq -e ".filesystems | length == 1"' 2024-04-02T04:56:12.610 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:12 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:12.610 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:12 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:12.610 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:12 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:56:12.610 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:12 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:56:12.610 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:12 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:12.610 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:12 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/537103280' entity='client.admin' cmd=[{"prefix": "fs dump"}]: dispatch 2024-04-02T04:56:12.610 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:12 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:56:12.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:12 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:12.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:12 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:12.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:12 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:56:12.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:12 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:56:12.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:12 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:12.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:12 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/537103280' entity='client.admin' cmd=[{"prefix": "fs dump"}]: dispatch 2024-04-02T04:56:12.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:12 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:56:12.994 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:56:13.416 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:13 smithi152 ceph-mon[29073]: pgmap v185: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 3.6 KiB/s rd, 716 B/s wr, 8 op/s 2024-04-02T04:56:13.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:13 smithi154 ceph-mon[35729]: pgmap v185: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 3.6 KiB/s rd, 716 B/s wr, 8 op/s 2024-04-02T04:56:14.574 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.569+0000 7f72195fd700 1 -- 172.21.15.152:0/1974001511 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f72140fcf80 msgr2=0x7f72140fd3f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:14.574 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.569+0000 7f72195fd700 1 --2- 172.21.15.152:0/1974001511 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f72140fcf80 0x7f72140fd3f0 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 crypto rx=0x7f71fc004940 tx=0x7f71fc009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:14.574 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.569+0000 7f72195fd700 1 -- 172.21.15.152:0/1974001511 shutdown_connections 2024-04-02T04:56:14.575 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.569+0000 7f72195fd700 1 --2- 172.21.15.152:0/1974001511 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f72140fcf80 0x7f72140fd3f0 unknown :-1 s=CLOSED pgs=248 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:14.575 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.569+0000 7f72195fd700 1 --2- 172.21.15.152:0/1974001511 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f72140fbcd0 0x7f72140fc0e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:14.575 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.569+0000 7f72195fd700 1 -- 172.21.15.152:0/1974001511 >> 172.21.15.152:0/1974001511 conn(0x7f72140f7450 msgr2=0x7f72140f9860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:14.575 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.569+0000 7f72195fd700 1 -- 172.21.15.152:0/1974001511 shutdown_connections 2024-04-02T04:56:14.575 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.569+0000 7f72195fd700 1 -- 172.21.15.152:0/1974001511 wait complete. 2024-04-02T04:56:14.575 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.569+0000 7f72195fd700 1 Processor -- start 2024-04-02T04:56:14.575 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.569+0000 7f72195fd700 1 -- start start 2024-04-02T04:56:14.575 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.569+0000 7f72195fd700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f72140fbcd0 0x7f721410d960 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:14.575 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.569+0000 7f72195fd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f72140fcf80 0x7f721410dea0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:14.575 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.569+0000 7f72195fd700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f721410e4a0 con 0x7f72140fcf80 2024-04-02T04:56:14.575 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.569+0000 7f72195fd700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f721410e610 con 0x7f72140fbcd0 2024-04-02T04:56:14.575 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.570+0000 7f720bfff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f72140fcf80 0x7f721410dea0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:14.575 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.570+0000 7f720bfff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f72140fcf80 0x7f721410dea0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:48554/0 (socket says 172.21.15.152:48554) 2024-04-02T04:56:14.576 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.570+0000 7f720bfff700 1 -- 172.21.15.152:0/475510409 learned_addr learned my addr 172.21.15.152:0/475510409 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:56:14.576 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.570+0000 7f7212ffd700 1 --2- 172.21.15.152:0/475510409 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f72140fbcd0 0x7f721410d960 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:14.576 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.570+0000 7f720bfff700 1 -- 172.21.15.152:0/475510409 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f72140fbcd0 msgr2=0x7f721410d960 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:14.576 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.570+0000 7f720bfff700 1 --2- 172.21.15.152:0/475510409 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f72140fbcd0 0x7f721410d960 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:14.576 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.570+0000 7f720bfff700 1 -- 172.21.15.152:0/475510409 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f71fc005040 con 0x7f72140fcf80 2024-04-02T04:56:14.576 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.570+0000 7f720bfff700 1 --2- 172.21.15.152:0/475510409 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f72140fcf80 0x7f721410dea0 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7f71fc00b3b0 tx=0x7f71fc009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:14.576 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.570+0000 7f720b7fe700 1 -- 172.21.15.152:0/475510409 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f71fc00e040 con 0x7f72140fcf80 2024-04-02T04:56:14.576 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.570+0000 7f720b7fe700 1 -- 172.21.15.152:0/475510409 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f71fc00bbf0 con 0x7f72140fcf80 2024-04-02T04:56:14.576 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.570+0000 7f72195fd700 1 -- 172.21.15.152:0/475510409 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7214112fc0 con 0x7f72140fcf80 2024-04-02T04:56:14.576 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.570+0000 7f720b7fe700 1 -- 172.21.15.152:0/475510409 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f71fc01c630 con 0x7f72140fcf80 2024-04-02T04:56:14.576 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.570+0000 7f72195fd700 1 -- 172.21.15.152:0/475510409 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7214113550 con 0x7f72140fcf80 2024-04-02T04:56:14.577 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.571+0000 7f72195fd700 1 -- 172.21.15.152:0/475510409 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7214101b10 con 0x7f72140fcf80 2024-04-02T04:56:14.577 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.572+0000 7f720b7fe700 1 -- 172.21.15.152:0/475510409 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f71fc00bd60 con 0x7f72140fcf80 2024-04-02T04:56:14.577 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.572+0000 7f720b7fe700 1 --2- 172.21.15.152:0/475510409 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f72000692a0 0x7f720006b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:14.578 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.572+0000 7f720b7fe700 1 -- 172.21.15.152:0/475510409 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7f71fc087ed0 con 0x7f72140fcf80 2024-04-02T04:56:14.578 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.572+0000 7f7212ffd700 1 --2- 172.21.15.152:0/475510409 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f72000692a0 0x7f720006b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:14.578 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.573+0000 7f7212ffd700 1 --2- 172.21.15.152:0/475510409 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f72000692a0 0x7f720006b750 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f7204004f40 tx=0x7f720400b040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:14.580 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.577+0000 7f720b7fe700 1 -- 172.21.15.152:0/475510409 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f71fc057c40 con 0x7f72140fcf80 2024-04-02T04:56:14.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:14 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:14.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:14 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:14.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:14 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:14.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:14 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:14.813 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.809+0000 7f72195fd700 1 -- 172.21.15.152:0/475510409 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7f721404c710 con 0x7f72140fcf80 2024-04-02T04:56:14.813 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.810+0000 7f720b7fe700 1 -- 172.21.15.152:0/475510409 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 20 v20) v1 ==== 94+0+4725 (secure 0 0 0) 0x7f71fc0577d0 con 0x7f72140fcf80 2024-04-02T04:56:14.816 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.812+0000 7f72195fd700 1 -- 172.21.15.152:0/475510409 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f72000692a0 msgr2=0x7f720006b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:14.816 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.812+0000 7f72195fd700 1 --2- 172.21.15.152:0/475510409 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f72000692a0 0x7f720006b750 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 crypto rx=0x7f7204004f40 tx=0x7f720400b040 comp rx=0 tx=0).stop 2024-04-02T04:56:14.816 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.812+0000 7f72195fd700 1 -- 172.21.15.152:0/475510409 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f72140fcf80 msgr2=0x7f721410dea0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:14.816 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.812+0000 7f72195fd700 1 --2- 172.21.15.152:0/475510409 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f72140fcf80 0x7f721410dea0 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 crypto rx=0x7f71fc00b3b0 tx=0x7f71fc009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:14.816 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.812+0000 7f72195fd700 1 -- 172.21.15.152:0/475510409 shutdown_connections 2024-04-02T04:56:14.816 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.812+0000 7f72195fd700 1 --2- 172.21.15.152:0/475510409 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f72000692a0 0x7f720006b750 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:14.816 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.812+0000 7f72195fd700 1 --2- 172.21.15.152:0/475510409 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f72140fbcd0 0x7f721410d960 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:14.816 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.812+0000 7f72195fd700 1 --2- 172.21.15.152:0/475510409 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f72140fcf80 0x7f721410dea0 unknown :-1 s=CLOSED pgs=249 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:14.816 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.812+0000 7f72195fd700 1 -- 172.21.15.152:0/475510409 >> 172.21.15.152:0/475510409 conn(0x7f72140f7450 msgr2=0x7f7214066850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:14.816 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.812+0000 7f72195fd700 1 -- 172.21.15.152:0/475510409 shutdown_connections 2024-04-02T04:56:14.817 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:14.812+0000 7f72195fd700 1 -- 172.21.15.152:0/475510409 wait complete. 2024-04-02T04:56:14.817 INFO:teuthology.orchestra.run.smithi152.stderr:dumped fsmap epoch 20 2024-04-02T04:56:14.826 INFO:teuthology.orchestra.run.smithi152.stdout:true 2024-04-02T04:56:15.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:15 smithi152 ceph-mon[29073]: pgmap v186: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 6.7 KiB/s rd, 0 B/s wr, 8 op/s 2024-04-02T04:56:15.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:15 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/475510409' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-02T04:56:15.940 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- bash -c 'while ! ceph --format=json mds versions | jq -e ". | add == 4"; do sleep 1; done' 2024-04-02T04:56:15.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:15 smithi154 ceph-mon[35729]: pgmap v186: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 6.7 KiB/s rd, 0 B/s wr, 8 op/s 2024-04-02T04:56:15.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:15 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/475510409' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-02T04:56:17.392 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:56:17.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:17 smithi152 ceph-mon[29073]: pgmap v187: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 5.8 KiB/s rd, 0 B/s wr, 7 op/s 2024-04-02T04:56:17.986 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:17 smithi154 ceph-mon[35729]: pgmap v187: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 5.8 KiB/s rd, 0 B/s wr, 7 op/s 2024-04-02T04:56:19.508 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:19 smithi152 ceph-mon[29073]: pgmap v188: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 5.6 KiB/s rd, 0 B/s wr, 7 op/s 2024-04-02T04:56:19.646 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.631+0000 7fc6f0ddf700 1 -- 172.21.15.152:0/1625397202 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc6ec0fed80 msgr2=0x7fc6ec0faec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:19.646 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.631+0000 7fc6f0ddf700 1 --2- 172.21.15.152:0/1625397202 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc6ec0fed80 0x7fc6ec0faec0 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 crypto rx=0x7fc6dc004940 tx=0x7fc6dc009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:19.646 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.631+0000 7fc6f0ddf700 1 -- 172.21.15.152:0/1625397202 shutdown_connections 2024-04-02T04:56:19.646 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.631+0000 7fc6f0ddf700 1 --2- 172.21.15.152:0/1625397202 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc6ec0fed80 0x7fc6ec0faec0 unknown :-1 s=CLOSED pgs=250 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:19.647 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.631+0000 7fc6f0ddf700 1 --2- 172.21.15.152:0/1625397202 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc6ec0fe430 0x7fc6ec0fe840 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:19.647 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.631+0000 7fc6f0ddf700 1 -- 172.21.15.152:0/1625397202 >> 172.21.15.152:0/1625397202 conn(0x7fc6ec0f6bd0 msgr2=0x7fc6ec0f9000 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:19.647 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.631+0000 7fc6f0ddf700 1 -- 172.21.15.152:0/1625397202 shutdown_connections 2024-04-02T04:56:19.647 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.631+0000 7fc6f0ddf700 1 -- 172.21.15.152:0/1625397202 wait complete. 2024-04-02T04:56:19.647 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.632+0000 7fc6f0ddf700 1 Processor -- start 2024-04-02T04:56:19.647 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.632+0000 7fc6f0ddf700 1 -- start start 2024-04-02T04:56:19.647 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.632+0000 7fc6f0ddf700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc6ec0fe430 0x7fc6ec0fda40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:19.647 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.632+0000 7fc6f0ddf700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc6ec0fed80 0x7fc6ec0fc0f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:19.647 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.632+0000 7fc6f0ddf700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc6ec0fe040 con 0x7fc6ec0fed80 2024-04-02T04:56:19.647 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.632+0000 7fc6f0ddf700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc6ec0fc630 con 0x7fc6ec0fe430 2024-04-02T04:56:19.647 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.632+0000 7fc6e9d9b700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc6ec0fed80 0x7fc6ec0fc0f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:19.647 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.632+0000 7fc6e9d9b700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc6ec0fed80 0x7fc6ec0fc0f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:48574/0 (socket says 172.21.15.152:48574) 2024-04-02T04:56:19.648 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.632+0000 7fc6e9d9b700 1 -- 172.21.15.152:0/1528294561 learned_addr learned my addr 172.21.15.152:0/1528294561 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:56:19.648 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.632+0000 7fc6ea59c700 1 --2- 172.21.15.152:0/1528294561 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc6ec0fe430 0x7fc6ec0fda40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:19.648 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.633+0000 7fc6e9d9b700 1 -- 172.21.15.152:0/1528294561 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc6ec0fe430 msgr2=0x7fc6ec0fda40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:19.648 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.633+0000 7fc6e9d9b700 1 --2- 172.21.15.152:0/1528294561 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc6ec0fe430 0x7fc6ec0fda40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:19.648 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.633+0000 7fc6e9d9b700 1 -- 172.21.15.152:0/1528294561 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc6dc005040 con 0x7fc6ec0fed80 2024-04-02T04:56:19.648 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.633+0000 7fc6e9d9b700 1 --2- 172.21.15.152:0/1528294561 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc6ec0fed80 0x7fc6ec0fc0f0 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7fc6dc00b3b0 tx=0x7fc6dc009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:19.648 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.633+0000 7fc6e2ffd700 1 -- 172.21.15.152:0/1528294561 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fc6dc00e040 con 0x7fc6ec0fed80 2024-04-02T04:56:19.648 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.633+0000 7fc6e2ffd700 1 -- 172.21.15.152:0/1528294561 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fc6dc00bcc0 con 0x7fc6ec0fed80 2024-04-02T04:56:19.648 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.633+0000 7fc6f0ddf700 1 -- 172.21.15.152:0/1528294561 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc6ec0fc880 con 0x7fc6ec0fed80 2024-04-02T04:56:19.648 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.633+0000 7fc6e2ffd700 1 -- 172.21.15.152:0/1528294561 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fc6dc01c630 con 0x7fc6ec0fed80 2024-04-02T04:56:19.648 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.633+0000 7fc6f0ddf700 1 -- 172.21.15.152:0/1528294561 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc6ec0fce40 con 0x7fc6ec0fed80 2024-04-02T04:56:19.648 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.634+0000 7fc6f0ddf700 1 -- 172.21.15.152:0/1528294561 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc6ec04c710 con 0x7fc6ec0fed80 2024-04-02T04:56:19.648 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.634+0000 7fc6e2ffd700 1 -- 172.21.15.152:0/1528294561 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7fc6dc013980 con 0x7fc6ec0fed80 2024-04-02T04:56:19.649 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.635+0000 7fc6e2ffd700 1 --2- 172.21.15.152:0/1528294561 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fc6d80692a0 0x7fc6d806b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:19.649 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.635+0000 7fc6e2ffd700 1 -- 172.21.15.152:0/1528294561 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7fc6dc088000 con 0x7fc6ec0fed80 2024-04-02T04:56:19.649 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.635+0000 7fc6ea59c700 1 --2- 172.21.15.152:0/1528294561 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fc6d80692a0 0x7fc6d806b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:19.649 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.635+0000 7fc6ea59c700 1 --2- 172.21.15.152:0/1528294561 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fc6d80692a0 0x7fc6d806b750 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7fc6d4004990 tx=0x7fc6d4009ee0 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:19.649 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.639+0000 7fc6e2ffd700 1 -- 172.21.15.152:0/1528294561 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fc6dc057cf0 con 0x7fc6ec0fed80 2024-04-02T04:56:19.948 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.943+0000 7fc6f0ddf700 1 -- 172.21.15.152:0/1528294561 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "mds versions", "format": "json"} v 0) v1 -- 0x7fc6ec1058e0 con 0x7fc6ec0fed80 2024-04-02T04:56:19.949 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.944+0000 7fc6e2ffd700 1 -- 172.21.15.152:0/1528294561 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "mds versions", "format": "json"}]=0 v20) v1 ==== 78+0+99 (secure 0 0 0) 0x7fc6dc057880 con 0x7fc6ec0fed80 2024-04-02T04:56:19.951 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.946+0000 7fc6f0ddf700 1 -- 172.21.15.152:0/1528294561 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fc6d80692a0 msgr2=0x7fc6d806b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:19.951 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.946+0000 7fc6f0ddf700 1 --2- 172.21.15.152:0/1528294561 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fc6d80692a0 0x7fc6d806b750 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 crypto rx=0x7fc6d4004990 tx=0x7fc6d4009ee0 comp rx=0 tx=0).stop 2024-04-02T04:56:19.951 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.946+0000 7fc6f0ddf700 1 -- 172.21.15.152:0/1528294561 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc6ec0fed80 msgr2=0x7fc6ec0fc0f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:19.951 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.946+0000 7fc6f0ddf700 1 --2- 172.21.15.152:0/1528294561 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc6ec0fed80 0x7fc6ec0fc0f0 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 crypto rx=0x7fc6dc00b3b0 tx=0x7fc6dc009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:19.951 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.946+0000 7fc6f0ddf700 1 -- 172.21.15.152:0/1528294561 shutdown_connections 2024-04-02T04:56:19.951 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.946+0000 7fc6f0ddf700 1 --2- 172.21.15.152:0/1528294561 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fc6d80692a0 0x7fc6d806b750 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:19.951 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.946+0000 7fc6f0ddf700 1 --2- 172.21.15.152:0/1528294561 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc6ec0fe430 0x7fc6ec0fda40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:19.951 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.946+0000 7fc6f0ddf700 1 --2- 172.21.15.152:0/1528294561 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc6ec0fed80 0x7fc6ec0fc0f0 unknown :-1 s=CLOSED pgs=251 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:19.951 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.946+0000 7fc6f0ddf700 1 -- 172.21.15.152:0/1528294561 >> 172.21.15.152:0/1528294561 conn(0x7fc6ec0f6bd0 msgr2=0x7fc6ec0f8210 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:19.952 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.946+0000 7fc6f0ddf700 1 -- 172.21.15.152:0/1528294561 shutdown_connections 2024-04-02T04:56:19.952 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:19.946+0000 7fc6f0ddf700 1 -- 172.21.15.152:0/1528294561 wait complete. 2024-04-02T04:56:19.961 INFO:teuthology.orchestra.run.smithi152.stdout:true 2024-04-02T04:56:19.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:19 smithi154 ceph-mon[35729]: pgmap v188: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 5.6 KiB/s rd, 0 B/s wr, 7 op/s 2024-04-02T04:56:20.574 INFO:teuthology.run_tasks:Running task fs.pre_upgrade_save... 2024-04-02T04:56:20.587 DEBUG:teuthology.orchestra.run.smithi152:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd dump --format=json 2024-04-02T04:56:20.693 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.689+0000 7fcd19018640 1 -- 172.21.15.152:0/3591310975 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcd140fd490 msgr2=0x7fcd140fd900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:20.694 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.689+0000 7fcd19018640 1 --2- 172.21.15.152:0/3591310975 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcd140fd490 0x7fcd140fd900 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 crypto rx=0x7fcd080048d0 tx=0x7fcd08009da0 comp rx=0 tx=0).stop 2024-04-02T04:56:20.694 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.689+0000 7fcd19018640 1 -- 172.21.15.152:0/3591310975 shutdown_connections 2024-04-02T04:56:20.694 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.689+0000 7fcd19018640 1 --2- 172.21.15.152:0/3591310975 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcd140fd490 0x7fcd140fd900 unknown :-1 s=CLOSED pgs=252 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:20.694 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.689+0000 7fcd19018640 1 --2- 172.21.15.152:0/3591310975 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcd140fc1c0 0x7fcd140fc5d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:20.694 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.689+0000 7fcd19018640 1 -- 172.21.15.152:0/3591310975 >> 172.21.15.152:0/3591310975 conn(0x7fcd140f7980 msgr2=0x7fcd140f9db0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:20.694 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.689+0000 7fcd19018640 1 -- 172.21.15.152:0/3591310975 shutdown_connections 2024-04-02T04:56:20.694 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.689+0000 7fcd19018640 1 -- 172.21.15.152:0/3591310975 wait complete. 2024-04-02T04:56:20.696 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.689+0000 7fcd19018640 1 Processor -- start 2024-04-02T04:56:20.696 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.689+0000 7fcd19018640 1 -- start start 2024-04-02T04:56:20.696 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.689+0000 7fcd19018640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcd140fc1c0 0x7fcd1410ccd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:20.696 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.689+0000 7fcd19018640 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcd140fd490 0x7fcd1410d210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:20.696 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.689+0000 7fcd19018640 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcd1410d890 con 0x7fcd140fd490 2024-04-02T04:56:20.696 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.689+0000 7fcd19018640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcd14112250 con 0x7fcd140fc1c0 2024-04-02T04:56:20.696 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.690+0000 7fcd1259c640 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcd140fd490 0x7fcd1410d210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:20.697 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.690+0000 7fcd1259c640 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcd140fd490 0x7fcd1410d210 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:37500/0 (socket says 172.21.15.152:37500) 2024-04-02T04:56:20.697 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.690+0000 7fcd1259c640 1 -- 172.21.15.152:0/2315409714 learned_addr learned my addr 172.21.15.152:0/2315409714 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:56:20.697 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.690+0000 7fcd12d9d640 1 --2- 172.21.15.152:0/2315409714 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcd140fc1c0 0x7fcd1410ccd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:20.697 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.690+0000 7fcd1259c640 1 -- 172.21.15.152:0/2315409714 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcd140fc1c0 msgr2=0x7fcd1410ccd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:20.697 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.690+0000 7fcd1259c640 1 --2- 172.21.15.152:0/2315409714 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcd140fc1c0 0x7fcd1410ccd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:20.697 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.690+0000 7fcd1259c640 1 -- 172.21.15.152:0/2315409714 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcd08005040 con 0x7fcd140fd490 2024-04-02T04:56:20.697 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.690+0000 7fcd1259c640 1 --2- 172.21.15.152:0/2315409714 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcd140fd490 0x7fcd1410d210 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7fcd08007dd0 tx=0x7fcd08009da0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:20.697 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.690+0000 7fccfb7fe640 1 -- 172.21.15.152:0/2315409714 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fcd080094c0 con 0x7fcd140fd490 2024-04-02T04:56:20.697 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.690+0000 7fcd19018640 1 -- 172.21.15.152:0/2315409714 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcd14112390 con 0x7fcd140fd490 2024-04-02T04:56:20.697 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.690+0000 7fccfb7fe640 1 -- 172.21.15.152:0/2315409714 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fcd08009620 con 0x7fcd140fd490 2024-04-02T04:56:20.697 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.690+0000 7fccfb7fe640 1 -- 172.21.15.152:0/2315409714 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fcd0803b640 con 0x7fcd140fd490 2024-04-02T04:56:20.697 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.690+0000 7fcd19018640 1 -- 172.21.15.152:0/2315409714 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcd14112880 con 0x7fcd140fd490 2024-04-02T04:56:20.697 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.691+0000 7fcd19018640 1 -- 172.21.15.152:0/2315409714 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcd1404bcc0 con 0x7fcd140fd490 2024-04-02T04:56:20.697 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.692+0000 7fccfb7fe640 1 -- 172.21.15.152:0/2315409714 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7fcd0802f070 con 0x7fcd140fd490 2024-04-02T04:56:20.698 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.692+0000 7fccfb7fe640 1 --2- 172.21.15.152:0/2315409714 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fccfc069210 0x7fccfc06b6c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:20.698 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.692+0000 7fccfb7fe640 1 -- 172.21.15.152:0/2315409714 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7fcd080a9730 con 0x7fcd140fd490 2024-04-02T04:56:20.698 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.692+0000 7fcd12d9d640 1 --2- 172.21.15.152:0/2315409714 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fccfc069210 0x7fccfc06b6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:20.698 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.693+0000 7fcd12d9d640 1 --2- 172.21.15.152:0/2315409714 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fccfc069210 0x7fccfc06b6c0 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7fcd140fa460 tx=0x7fcd0400b040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:20.701 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.697+0000 7fccfb7fe640 1 -- 172.21.15.152:0/2315409714 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fcd08075990 con 0x7fcd140fd490 2024-04-02T04:56:20.871 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.866+0000 7fcd19018640 1 -- 172.21.15.152:0/2315409714 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "osd dump", "format": "json"} v 0) v1 -- 0x7fcd14104d30 con 0x7fcd140fd490 2024-04-02T04:56:20.871 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.867+0000 7fccfb7fe640 1 -- 172.21.15.152:0/2315409714 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "osd dump", "format": "json"}]=0 v46) v1 ==== 74+0+14429 (secure 0 0 0) 0x7fcd08078fb0 con 0x7fcd140fd490 2024-04-02T04:56:20.872 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:56:20.872 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":46,"fsid":"2e0055d0-f0ac-11ee-b647-cb9ed24678a4","created":"2024-04-02T04:48:59.746166+0000","modified":"2024-04-02T04:56:04.513029+0000","last_up_change":"2024-04-02T04:54:32.119050+0000","last_in_change":"2024-04-02T04:54:13.064706+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":14,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":6,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-02T04:53:20.274250+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs.cephfs.meta","create_time":"2024-04-02T04:55:48.943469+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"44","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs.cephfs.data","create_time":"2024-04-02T04:55:49.671860+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"45","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"3f25c359-f643-4dfb-ba7b-30bb1a2dfcb2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6802","nonce":718734380},{"type":"v1","addr":"172.21.15.152:6803","nonce":718734380}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6804","nonce":718734380},{"type":"v1","addr":"172.21.15.152:6805","nonce":718734380}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6808","nonce":718734380},{"type":"v1","addr":"172.21.15.152:6809","nonce":718734380}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6806","nonce":718734380},{"type":"v1","addr":"172.21.15.152:6807","nonce":718734380}]},"public_addr":"172.21.15.152:6803/718734380","cluster_addr":"172.21.15.152:6805/718734380","heartbeat_back_addr":"172.21.15.152:6809/718734380","heartbeat_front_addr":"172.21.15.152:6807/718734380","state":["exists","up"]},{"osd":1,"uuid":"d4555f9e-fc0e-4266-ae65-3c197d3fa02c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":14,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6810","nonce":782616861},{"type":"v1","addr":"172.21.15.152:6811","nonce":782616861}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6812","nonce":782616861},{"type":"v1","addr":"172.21.15.152:6813","nonce":782616861}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6816","nonce":782616861},{"type":"v1","addr":"172.21.15.152:6817","nonce":782616861}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6814","nonce":782616861},{"type":"v1","addr":"172.21.15.152:6815","nonce":782616861}]},"public_addr":"172.21.15.152:6811/782616861","cluster_addr":"172.21.15.152:6813/782616861","heartbeat_back_addr":"172.21.15.152:6817/782616861","heartbeat_front_addr":"172.21.15.152:6815/782616861","state":["exists","up"]},{"osd":2,"uuid":"caa0a912-a9d5-45e7-afe2-51beaf0f1742","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":19,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6818","nonce":1566571469},{"type":"v1","addr":"172.21.15.152:6819","nonce":1566571469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6820","nonce":1566571469},{"type":"v1","addr":"172.21.15.152:6821","nonce":1566571469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6824","nonce":1566571469},{"type":"v1","addr":"172.21.15.152:6825","nonce":1566571469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6822","nonce":1566571469},{"type":"v1","addr":"172.21.15.152:6823","nonce":1566571469}]},"public_addr":"172.21.15.152:6819/1566571469","cluster_addr":"172.21.15.152:6821/1566571469","heartbeat_back_addr":"172.21.15.152:6825/1566571469","heartbeat_front_addr":"172.21.15.152:6823/1566571469","state":["exists","up"]},{"osd":3,"uuid":"84a2d337-6ac8-41e4-a922-a6e94c1e9067","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6800","nonce":3366219775},{"type":"v1","addr":"172.21.15.154:6801","nonce":3366219775}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6802","nonce":3366219775},{"type":"v1","addr":"172.21.15.154:6803","nonce":3366219775}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6806","nonce":3366219775},{"type":"v1","addr":"172.21.15.154:6807","nonce":3366219775}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6804","nonce":3366219775},{"type":"v1","addr":"172.21.15.154:6805","nonce":3366219775}]},"public_addr":"172.21.15.154:6801/3366219775","cluster_addr":"172.21.15.154:6803/3366219775","heartbeat_back_addr":"172.21.15.154:6807/3366219775","heartbeat_front_addr":"172.21.15.154:6805/3366219775","state":["exists","up"]},{"osd":4,"uuid":"d547308c-898c-4165-9762-c8fd4170dc39","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":34,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6808","nonce":220818867},{"type":"v1","addr":"172.21.15.154:6809","nonce":220818867}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6810","nonce":220818867},{"type":"v1","addr":"172.21.15.154:6811","nonce":220818867}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6814","nonce":220818867},{"type":"v1","addr":"172.21.15.154:6815","nonce":220818867}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6812","nonce":220818867},{"type":"v1","addr":"172.21.15.154:6813","nonce":220818867}]},"public_addr":"172.21.15.154:6809/220818867","cluster_addr":"172.21.15.154:6811/220818867","heartbeat_back_addr":"172.21.15.154:6815/220818867","heartbeat_front_addr":"172.21.15.154:6813/220818867","state":["exists","up"]},{"osd":5,"uuid":"c3d3760a-08a7-44f2-8288-ccc5d08df057","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6816","nonce":1178927498},{"type":"v1","addr":"172.21.15.154:6817","nonce":1178927498}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6818","nonce":1178927498},{"type":"v1","addr":"172.21.15.154:6819","nonce":1178927498}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6822","nonce":1178927498},{"type":"v1","addr":"172.21.15.154:6823","nonce":1178927498}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6820","nonce":1178927498},{"type":"v1","addr":"172.21.15.154:6821","nonce":1178927498}]},"public_addr":"172.21.15.154:6817/1178927498","cluster_addr":"172.21.15.154:6819/1178927498","heartbeat_back_addr":"172.21.15.154:6823/1178927498","heartbeat_front_addr":"172.21.15.154:6821/1178927498","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:52:20.472492+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:52:49.979314+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:53:17.640952+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:53:40.392841+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:54:05.761710+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-04-02T04:54:30.235521+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.154:6825/1663663935":"2024-04-03T04:56:04.513001+0000","172.21.15.152:0/2947158817":"2024-04-03T04:50:01.902008+0000","172.21.15.152:6801/2913738627":"2024-04-03T04:49:30.819772+0000","172.21.15.152:0/3023512755":"2024-04-03T04:49:30.819772+0000","172.21.15.152:6800/3544077480":"2024-04-03T04:50:01.902008+0000","172.21.15.152:6800/2913738627":"2024-04-03T04:49:30.819772+0000","172.21.15.152:0/3476288520":"2024-04-03T04:49:30.819772+0000","172.21.15.152:6801/3544077480":"2024-04-03T04:50:01.902008+0000","172.21.15.152:0/2491428539":"2024-04-03T04:51:09.949838+0000","172.21.15.152:0/1943568671":"2024-04-03T04:49:30.819772+0000","172.21.15.152:0/1860988983":"2024-04-03T04:50:01.902008+0000","172.21.15.152:0/4188679877":"2024-04-03T04:50:01.902008+0000","172.21.15.152:6800/2497421829":"2024-04-03T04:51:09.949838+0000","172.21.15.152:6801/2497421829":"2024-04-03T04:51:09.949838+0000","172.21.15.152:0/2268166004":"2024-04-03T04:51:09.949838+0000","172.21.15.152:0/3027168373":"2024-04-03T04:51:09.949838+0000","172.21.15.154:6824/1663663935":"2024-04-03T04:56:04.513001+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-02T04:56:20.874 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.869+0000 7fcd19018640 1 -- 172.21.15.152:0/2315409714 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fccfc069210 msgr2=0x7fccfc06b6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:20.874 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.869+0000 7fcd19018640 1 --2- 172.21.15.152:0/2315409714 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fccfc069210 0x7fccfc06b6c0 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 crypto rx=0x7fcd140fa460 tx=0x7fcd0400b040 comp rx=0 tx=0).stop 2024-04-02T04:56:20.874 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.869+0000 7fcd19018640 1 -- 172.21.15.152:0/2315409714 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcd140fd490 msgr2=0x7fcd1410d210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:20.874 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.869+0000 7fcd19018640 1 --2- 172.21.15.152:0/2315409714 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcd140fd490 0x7fcd1410d210 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 crypto rx=0x7fcd08007dd0 tx=0x7fcd08009da0 comp rx=0 tx=0).stop 2024-04-02T04:56:20.874 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.869+0000 7fcd19018640 1 -- 172.21.15.152:0/2315409714 shutdown_connections 2024-04-02T04:56:20.874 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.869+0000 7fcd19018640 1 --2- 172.21.15.152:0/2315409714 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fccfc069210 0x7fccfc06b6c0 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:20.874 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.869+0000 7fcd19018640 1 --2- 172.21.15.152:0/2315409714 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fcd140fd490 0x7fcd1410d210 unknown :-1 s=CLOSED pgs=253 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:20.874 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.869+0000 7fcd19018640 1 --2- 172.21.15.152:0/2315409714 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fcd140fc1c0 0x7fcd1410ccd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:20.874 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.869+0000 7fcd19018640 1 -- 172.21.15.152:0/2315409714 >> 172.21.15.152:0/2315409714 conn(0x7fcd140f7980 msgr2=0x7fcd140f9020 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:20.874 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.869+0000 7fcd19018640 1 -- 172.21.15.152:0/2315409714 shutdown_connections 2024-04-02T04:56:20.875 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.869+0000 7fcd19018640 1 -- 172.21.15.152:0/2315409714 wait complete. 2024-04-02T04:56:20.885 INFO:tasks.cephfs.filesystem.ceph_manager:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-02T04:53:20.274250+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}, {'pool': 2, 'pool_name': 'cephfs.cephfs.meta', 'create_time': '2024-04-02T04:55:48.943469+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '44', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_autoscale_bias': 4, 'pg_num_min': 16, 'recovery_priority': 5}, 'application_metadata': {'cephfs': {'metadata': 'cephfs'}}}, {'pool': 3, 'pool_name': 'cephfs.cephfs.data', 'create_time': '2024-04-02T04:55:49.671860+0000', 'flags': 131073, 'flags_names': 'hashpspool,bulk', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 32, 'pg_placement_num': 32, 'pg_placement_num_target': 32, 'pg_num_target': 32, 'pg_num_pending': 32, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '45', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {}, 'application_metadata': {'cephfs': {'data': 'cephfs'}}}] 2024-04-02T04:56:20.886 DEBUG:teuthology.orchestra.run.smithi152:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get .mgr pg_num 2024-04-02T04:56:20.988 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.982+0000 7f51cdb5d640 1 -- 172.21.15.152:0/1596575947 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f51c80ff2f0 msgr2=0x7f51c81016d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:20.988 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.982+0000 7f51cdb5d640 1 --2- 172.21.15.152:0/1596575947 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f51c80ff2f0 0x7f51c81016d0 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 crypto rx=0x7f51b00048d0 tx=0x7f51b0009da0 comp rx=0 tx=0).stop 2024-04-02T04:56:20.989 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.982+0000 7f51cdb5d640 1 -- 172.21.15.152:0/1596575947 shutdown_connections 2024-04-02T04:56:20.989 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.982+0000 7f51cdb5d640 1 --2- 172.21.15.152:0/1596575947 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f51c80ff2f0 0x7f51c81016d0 unknown :-1 s=CLOSED pgs=254 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:20.989 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.982+0000 7f51cdb5d640 1 --2- 172.21.15.152:0/1596575947 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f51c8066210 0x7f51c80fedb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:20.989 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.982+0000 7f51cdb5d640 1 -- 172.21.15.152:0/1596575947 >> 172.21.15.152:0/1596575947 conn(0x7f51c80f6880 msgr2=0x7f51c80f8c90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:20.989 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.982+0000 7f51cdb5d640 1 -- 172.21.15.152:0/1596575947 shutdown_connections 2024-04-02T04:56:20.989 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.982+0000 7f51cdb5d640 1 -- 172.21.15.152:0/1596575947 wait complete. 2024-04-02T04:56:20.989 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.982+0000 7f51cdb5d640 1 Processor -- start 2024-04-02T04:56:20.989 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.982+0000 7f51cdb5d640 1 -- start start 2024-04-02T04:56:20.989 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.983+0000 7f51cdb5d640 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f51c8066210 0x7f51c810cd50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:20.989 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.983+0000 7f51cdb5d640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f51c80ff2f0 0x7f51c810d290 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:20.989 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.983+0000 7f51cdb5d640 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f51c810d8e0 con 0x7f51c8066210 2024-04-02T04:56:20.989 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.983+0000 7f51cdb5d640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f51c8112250 con 0x7f51c80ff2f0 2024-04-02T04:56:20.989 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.983+0000 7f51c77fe640 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f51c8066210 0x7f51c810cd50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:20.989 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.983+0000 7f51c77fe640 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f51c8066210 0x7f51c810cd50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:37518/0 (socket says 172.21.15.152:37518) 2024-04-02T04:56:20.990 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.983+0000 7f51c77fe640 1 -- 172.21.15.152:0/3414081345 learned_addr learned my addr 172.21.15.152:0/3414081345 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:56:20.990 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.983+0000 7f51beffd640 1 --2- 172.21.15.152:0/3414081345 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f51c80ff2f0 0x7f51c810d290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:20.990 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.983+0000 7f51c77fe640 1 -- 172.21.15.152:0/3414081345 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f51c80ff2f0 msgr2=0x7f51c810d290 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:20.990 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.983+0000 7f51c77fe640 1 --2- 172.21.15.152:0/3414081345 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f51c80ff2f0 0x7f51c810d290 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:20.990 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.983+0000 7f51c77fe640 1 -- 172.21.15.152:0/3414081345 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f51b0005040 con 0x7f51c8066210 2024-04-02T04:56:20.990 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.983+0000 7f51c77fe640 1 --2- 172.21.15.152:0/3414081345 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f51c8066210 0x7f51c810cd50 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7f51b8002840 tx=0x7f51b8004570 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:20.990 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.983+0000 7f51c4ff9640 1 -- 172.21.15.152:0/3414081345 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f51b8007c30 con 0x7f51c8066210 2024-04-02T04:56:20.990 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.983+0000 7f51cdb5d640 1 -- 172.21.15.152:0/3414081345 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f51c81123f0 con 0x7f51c8066210 2024-04-02T04:56:20.990 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.983+0000 7f51c4ff9640 1 -- 172.21.15.152:0/3414081345 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f51b800f040 con 0x7f51c8066210 2024-04-02T04:56:20.990 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.983+0000 7f51c4ff9640 1 -- 172.21.15.152:0/3414081345 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f51b8009570 con 0x7f51c8066210 2024-04-02T04:56:20.990 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.983+0000 7f51cdb5d640 1 -- 172.21.15.152:0/3414081345 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f51c81128c0 con 0x7f51c8066210 2024-04-02T04:56:20.991 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.984+0000 7f51cdb5d640 1 -- 172.21.15.152:0/3414081345 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f51c804bcc0 con 0x7f51c8066210 2024-04-02T04:56:20.991 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.985+0000 7f51c4ff9640 1 -- 172.21.15.152:0/3414081345 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f51b8009750 con 0x7f51c8066210 2024-04-02T04:56:20.991 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.985+0000 7f51c4ff9640 1 --2- 172.21.15.152:0/3414081345 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f51b4069210 0x7f51b406b6c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:20.991 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.985+0000 7f51c4ff9640 1 -- 172.21.15.152:0/3414081345 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7f51b80870b0 con 0x7f51c8066210 2024-04-02T04:56:20.991 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.985+0000 7f51beffd640 1 --2- 172.21.15.152:0/3414081345 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f51b4069210 0x7f51b406b6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:20.992 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.985+0000 7f51beffd640 1 --2- 172.21.15.152:0/3414081345 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f51b4069210 0x7f51b406b6c0 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f51b00317e0 tx=0x7f51b0009da0 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:20.994 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:20.989+0000 7f51c4ff9640 1 -- 172.21.15.152:0/3414081345 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f51b8053310 con 0x7f51c8066210 2024-04-02T04:56:21.160 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.155+0000 7f51cdb5d640 1 -- 172.21.15.152:0/3414081345 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"} v 0) v1 -- 0x7f51c8112ae0 con 0x7f51c8066210 2024-04-02T04:56:21.160 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.156+0000 7f51c4ff9640 1 -- 172.21.15.152:0/3414081345 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]=0 v46) v1 ==== 93+0+10 (secure 0 0 0) 0x7f51c8112ae0 con 0x7f51c8066210 2024-04-02T04:56:21.160 INFO:teuthology.orchestra.run.smithi152.stdout:pg_num: 1 2024-04-02T04:56:21.162 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.157+0000 7f51cdb5d640 1 -- 172.21.15.152:0/3414081345 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f51b4069210 msgr2=0x7f51b406b6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:21.162 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.157+0000 7f51cdb5d640 1 --2- 172.21.15.152:0/3414081345 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f51b4069210 0x7f51b406b6c0 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 crypto rx=0x7f51b00317e0 tx=0x7f51b0009da0 comp rx=0 tx=0).stop 2024-04-02T04:56:21.163 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.157+0000 7f51cdb5d640 1 -- 172.21.15.152:0/3414081345 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f51c8066210 msgr2=0x7f51c810cd50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:21.163 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.157+0000 7f51cdb5d640 1 --2- 172.21.15.152:0/3414081345 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f51c8066210 0x7f51c810cd50 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 crypto rx=0x7f51b8002840 tx=0x7f51b8004570 comp rx=0 tx=0).stop 2024-04-02T04:56:21.163 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.157+0000 7f51cdb5d640 1 -- 172.21.15.152:0/3414081345 shutdown_connections 2024-04-02T04:56:21.163 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.157+0000 7f51cdb5d640 1 --2- 172.21.15.152:0/3414081345 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f51b4069210 0x7f51b406b6c0 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:21.163 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.157+0000 7f51cdb5d640 1 --2- 172.21.15.152:0/3414081345 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f51c80ff2f0 0x7f51c810d290 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:21.163 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.157+0000 7f51cdb5d640 1 --2- 172.21.15.152:0/3414081345 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f51c8066210 0x7f51c810cd50 unknown :-1 s=CLOSED pgs=255 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:21.163 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.157+0000 7f51cdb5d640 1 -- 172.21.15.152:0/3414081345 >> 172.21.15.152:0/3414081345 conn(0x7f51c80f6880 msgr2=0x7f51c80f6c50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:21.163 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.158+0000 7f51cdb5d640 1 -- 172.21.15.152:0/3414081345 shutdown_connections 2024-04-02T04:56:21.163 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.158+0000 7f51cdb5d640 1 -- 172.21.15.152:0/3414081345 wait complete. 2024-04-02T04:56:21.172 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:20 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:21.172 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:20 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:21.172 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:20 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:56:21.172 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:20 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:56:21.173 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:20 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:21.173 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:20 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/1528294561' entity='client.admin' cmd=[{"prefix": "mds versions", "format": "json"}]: dispatch 2024-04-02T04:56:21.173 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:20 smithi152 ceph-mon[29073]: pgmap v189: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 5.8 KiB/s rd, 0 B/s wr, 7 op/s 2024-04-02T04:56:21.174 DEBUG:teuthology.orchestra.run.smithi152:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs.cephfs.meta pg_num 2024-04-02T04:56:21.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:20 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:21.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:20 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:21.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:20 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:56:21.234 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:20 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:56:21.234 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:20 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:21.234 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:20 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/1528294561' entity='client.admin' cmd=[{"prefix": "mds versions", "format": "json"}]: dispatch 2024-04-02T04:56:21.234 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:20 smithi154 ceph-mon[35729]: pgmap v189: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 5.8 KiB/s rd, 0 B/s wr, 7 op/s 2024-04-02T04:56:21.276 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.269+0000 7f6528aae640 1 -- 172.21.15.152:0/2806614226 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f65240fe140 msgr2=0x7f65240fa2a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:21.276 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.269+0000 7f6528aae640 1 --2- 172.21.15.152:0/2806614226 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f65240fe140 0x7f65240fa2a0 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 crypto rx=0x7f650c0048d0 tx=0x7f650c009da0 comp rx=0 tx=0).stop 2024-04-02T04:56:21.276 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.269+0000 7f6528aae640 1 -- 172.21.15.152:0/2806614226 shutdown_connections 2024-04-02T04:56:21.276 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.269+0000 7f6528aae640 1 --2- 172.21.15.152:0/2806614226 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f65240fe140 0x7f65240fa2a0 unknown :-1 s=CLOSED pgs=256 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:21.276 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.269+0000 7f6528aae640 1 --2- 172.21.15.152:0/2806614226 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f65240fd7f0 0x7f65240fdc00 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:21.276 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.269+0000 7f6528aae640 1 -- 172.21.15.152:0/2806614226 >> 172.21.15.152:0/2806614226 conn(0x7f65240f5ff0 msgr2=0x7f65240f8420 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:21.276 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.269+0000 7f6528aae640 1 -- 172.21.15.152:0/2806614226 shutdown_connections 2024-04-02T04:56:21.276 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.269+0000 7f6528aae640 1 -- 172.21.15.152:0/2806614226 wait complete. 2024-04-02T04:56:21.277 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.269+0000 7f6528aae640 1 Processor -- start 2024-04-02T04:56:21.277 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.269+0000 7f6528aae640 1 -- start start 2024-04-02T04:56:21.277 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.269+0000 7f6528aae640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f65240fd7f0 0x7f652410cd10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:21.277 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.270+0000 7f6528aae640 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f65240fe140 0x7f652410d250 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:21.277 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.270+0000 7f6528aae640 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f652410d8a0 con 0x7f65240fe140 2024-04-02T04:56:21.277 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.270+0000 7f6528aae640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6524112260 con 0x7f65240fd7f0 2024-04-02T04:56:21.277 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.270+0000 7f651bfff640 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f65240fe140 0x7f652410d250 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:21.277 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.270+0000 7f651bfff640 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f65240fe140 0x7f652410d250 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:37536/0 (socket says 172.21.15.152:37536) 2024-04-02T04:56:21.277 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.270+0000 7f651bfff640 1 -- 172.21.15.152:0/4162559691 learned_addr learned my addr 172.21.15.152:0/4162559691 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:56:21.277 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.270+0000 7f652259c640 1 --2- 172.21.15.152:0/4162559691 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f65240fd7f0 0x7f652410cd10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:21.277 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.270+0000 7f651bfff640 1 -- 172.21.15.152:0/4162559691 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f65240fd7f0 msgr2=0x7f652410cd10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:21.277 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.270+0000 7f651bfff640 1 --2- 172.21.15.152:0/4162559691 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f65240fd7f0 0x7f652410cd10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:21.277 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.270+0000 7f651bfff640 1 -- 172.21.15.152:0/4162559691 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f650c005040 con 0x7f65240fe140 2024-04-02T04:56:21.278 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.270+0000 7f652259c640 1 --2- 172.21.15.152:0/4162559691 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f65240fd7f0 0x7f652410cd10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-02T04:56:21.278 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.270+0000 7f651bfff640 1 --2- 172.21.15.152:0/4162559691 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f65240fe140 0x7f652410d250 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7f650c000c00 tx=0x7f650c009da0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:21.278 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.270+0000 7f651affd640 1 -- 172.21.15.152:0/4162559691 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f650c0094c0 con 0x7f65240fe140 2024-04-02T04:56:21.278 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.270+0000 7f651affd640 1 -- 172.21.15.152:0/4162559691 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f650c009620 con 0x7f65240fe140 2024-04-02T04:56:21.278 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.270+0000 7f6528aae640 1 -- 172.21.15.152:0/4162559691 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f65241123a0 con 0x7f65240fe140 2024-04-02T04:56:21.278 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.270+0000 7f651affd640 1 -- 172.21.15.152:0/4162559691 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f650c0435f0 con 0x7f65240fe140 2024-04-02T04:56:21.278 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.270+0000 7f6528aae640 1 -- 172.21.15.152:0/4162559691 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6524112810 con 0x7f65240fe140 2024-04-02T04:56:21.278 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.271+0000 7f6528aae640 1 -- 172.21.15.152:0/4162559691 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f652404bcc0 con 0x7f65240fe140 2024-04-02T04:56:21.279 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.272+0000 7f651affd640 1 -- 172.21.15.152:0/4162559691 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f650c02f070 con 0x7f65240fe140 2024-04-02T04:56:21.279 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.272+0000 7f651affd640 1 --2- 172.21.15.152:0/4162559691 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f6510069220 0x7f651006b6d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:21.279 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.272+0000 7f652259c640 1 --2- 172.21.15.152:0/4162559691 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f6510069220 0x7f651006b6d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:21.279 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.272+0000 7f651affd640 1 -- 172.21.15.152:0/4162559691 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7f650c0a9270 con 0x7f65240fe140 2024-04-02T04:56:21.279 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.273+0000 7f652259c640 1 --2- 172.21.15.152:0/4162559691 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f6510069220 0x7f651006b6d0 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f65140041d0 tx=0x7f6514009040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:21.281 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.276+0000 7f651affd640 1 -- 172.21.15.152:0/4162559691 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f650c0754d0 con 0x7f65240fe140 2024-04-02T04:56:21.447 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.443+0000 7f6528aae640 1 -- 172.21.15.152:0/4162559691 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs.cephfs.meta", "var": "pg_num"} v 0) v1 -- 0x7f6524112a30 con 0x7f65240fe140 2024-04-02T04:56:21.448 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.443+0000 7f651affd640 1 -- 172.21.15.152:0/4162559691 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs.cephfs.meta", "var": "pg_num"}]=0 v46) v1 ==== 107+0+11 (secure 0 0 0) 0x7f650c078af0 con 0x7f65240fe140 2024-04-02T04:56:21.448 INFO:teuthology.orchestra.run.smithi152.stdout:pg_num: 32 2024-04-02T04:56:21.450 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.445+0000 7f6528aae640 1 -- 172.21.15.152:0/4162559691 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f6510069220 msgr2=0x7f651006b6d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:21.450 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.445+0000 7f6528aae640 1 --2- 172.21.15.152:0/4162559691 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f6510069220 0x7f651006b6d0 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 crypto rx=0x7f65140041d0 tx=0x7f6514009040 comp rx=0 tx=0).stop 2024-04-02T04:56:21.450 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.445+0000 7f6528aae640 1 -- 172.21.15.152:0/4162559691 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f65240fe140 msgr2=0x7f652410d250 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:21.450 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.445+0000 7f6528aae640 1 --2- 172.21.15.152:0/4162559691 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f65240fe140 0x7f652410d250 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 crypto rx=0x7f650c000c00 tx=0x7f650c009da0 comp rx=0 tx=0).stop 2024-04-02T04:56:21.450 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.445+0000 7f6528aae640 1 -- 172.21.15.152:0/4162559691 shutdown_connections 2024-04-02T04:56:21.450 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.445+0000 7f6528aae640 1 --2- 172.21.15.152:0/4162559691 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f6510069220 0x7f651006b6d0 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:21.450 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.445+0000 7f6528aae640 1 --2- 172.21.15.152:0/4162559691 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f65240fe140 0x7f652410d250 unknown :-1 s=CLOSED pgs=257 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:21.451 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.445+0000 7f6528aae640 1 --2- 172.21.15.152:0/4162559691 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f65240fd7f0 0x7f652410cd10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:21.451 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.445+0000 7f6528aae640 1 -- 172.21.15.152:0/4162559691 >> 172.21.15.152:0/4162559691 conn(0x7f65240f5ff0 msgr2=0x7f65240f76a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:21.451 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.445+0000 7f6528aae640 1 -- 172.21.15.152:0/4162559691 shutdown_connections 2024-04-02T04:56:21.451 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.445+0000 7f6528aae640 1 -- 172.21.15.152:0/4162559691 wait complete. 2024-04-02T04:56:21.461 DEBUG:teuthology.orchestra.run.smithi152:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd pool get cephfs.cephfs.data pg_num 2024-04-02T04:56:21.563 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.557+0000 7f28b0225640 1 -- 172.21.15.152:0/131580857 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f28a80fe150 msgr2=0x7f28a80fa2b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:21.563 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.557+0000 7f28b0225640 1 --2- 172.21.15.152:0/131580857 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f28a80fe150 0x7f28a80fa2b0 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 crypto rx=0x7f28980048d0 tx=0x7f2898009da0 comp rx=0 tx=0).stop 2024-04-02T04:56:21.564 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.557+0000 7f28b0225640 1 -- 172.21.15.152:0/131580857 shutdown_connections 2024-04-02T04:56:21.564 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.557+0000 7f28b0225640 1 --2- 172.21.15.152:0/131580857 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f28a80fe150 0x7f28a80fa2b0 unknown :-1 s=CLOSED pgs=258 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:21.564 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.557+0000 7f28b0225640 1 --2- 172.21.15.152:0/131580857 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f28a80fd800 0x7f28a80fdc10 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:21.564 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.557+0000 7f28b0225640 1 -- 172.21.15.152:0/131580857 >> 172.21.15.152:0/131580857 conn(0x7f28a80f6000 msgr2=0x7f28a80f8430 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:21.564 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.557+0000 7f28b0225640 1 -- 172.21.15.152:0/131580857 shutdown_connections 2024-04-02T04:56:21.564 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.557+0000 7f28b0225640 1 -- 172.21.15.152:0/131580857 wait complete. 2024-04-02T04:56:21.564 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.558+0000 7f28b0225640 1 Processor -- start 2024-04-02T04:56:21.564 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.558+0000 7f28b0225640 1 -- start start 2024-04-02T04:56:21.564 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.558+0000 7f28b0225640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f28a80fd800 0x7f28a810cd00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:21.564 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.558+0000 7f28b0225640 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f28a80fe150 0x7f28a810d240 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:21.564 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.558+0000 7f28b0225640 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28a810d890 con 0x7f28a80fe150 2024-04-02T04:56:21.564 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.558+0000 7f28b0225640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28a8112250 con 0x7f28a80fd800 2024-04-02T04:56:21.564 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.558+0000 7f28ad7c0640 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f28a80fe150 0x7f28a810d240 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:21.564 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.558+0000 7f28ad7c0640 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f28a80fe150 0x7f28a810d240 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:37564/0 (socket says 172.21.15.152:37564) 2024-04-02T04:56:21.565 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.558+0000 7f28ad7c0640 1 -- 172.21.15.152:0/2346293981 learned_addr learned my addr 172.21.15.152:0/2346293981 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:56:21.565 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.558+0000 7f28adfc1640 1 --2- 172.21.15.152:0/2346293981 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f28a80fd800 0x7f28a810cd00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:21.565 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.558+0000 7f28ad7c0640 1 -- 172.21.15.152:0/2346293981 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f28a80fd800 msgr2=0x7f28a810cd00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:21.565 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.558+0000 7f28ad7c0640 1 --2- 172.21.15.152:0/2346293981 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f28a80fd800 0x7f28a810cd00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:21.565 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.558+0000 7f28ad7c0640 1 -- 172.21.15.152:0/2346293981 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2898005040 con 0x7f28a80fe150 2024-04-02T04:56:21.565 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.558+0000 7f28ad7c0640 1 --2- 172.21.15.152:0/2346293981 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f28a80fe150 0x7f28a810d240 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f289802bda0 tx=0x7f2898009da0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:21.565 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.559+0000 7f289e7fc640 1 -- 172.21.15.152:0/2346293981 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f28980094c0 con 0x7f28a80fe150 2024-04-02T04:56:21.565 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.559+0000 7f289e7fc640 1 -- 172.21.15.152:0/2346293981 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f2898009620 con 0x7f28a80fe150 2024-04-02T04:56:21.565 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.559+0000 7f28b0225640 1 -- 172.21.15.152:0/2346293981 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f28a8112390 con 0x7f28a80fe150 2024-04-02T04:56:21.565 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.559+0000 7f289e7fc640 1 -- 172.21.15.152:0/2346293981 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f289803b640 con 0x7f28a80fe150 2024-04-02T04:56:21.565 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.559+0000 7f28b0225640 1 -- 172.21.15.152:0/2346293981 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f28a8112860 con 0x7f28a80fe150 2024-04-02T04:56:21.566 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.560+0000 7f28b0225640 1 -- 172.21.15.152:0/2346293981 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f28a804bcc0 con 0x7f28a80fe150 2024-04-02T04:56:21.566 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.560+0000 7f289e7fc640 1 -- 172.21.15.152:0/2346293981 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f289802f070 con 0x7f28a80fe150 2024-04-02T04:56:21.566 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.560+0000 7f289e7fc640 1 --2- 172.21.15.152:0/2346293981 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2894069210 0x7f289406b6c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:21.566 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.560+0000 7f289e7fc640 1 -- 172.21.15.152:0/2346293981 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7f28980a9720 con 0x7f28a80fe150 2024-04-02T04:56:21.566 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.561+0000 7f28adfc1640 1 --2- 172.21.15.152:0/2346293981 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2894069210 0x7f289406b6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:21.566 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.561+0000 7f28adfc1640 1 --2- 172.21.15.152:0/2346293981 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2894069210 0x7f289406b6c0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f28a810e1c0 tx=0x7f28a400b040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:21.569 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.565+0000 7f289e7fc640 1 -- 172.21.15.152:0/2346293981 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f2898075980 con 0x7f28a80fe150 2024-04-02T04:56:21.735 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.730+0000 7f28b0225640 1 -- 172.21.15.152:0/2346293981 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "osd pool get", "pool": "cephfs.cephfs.data", "var": "pg_num"} v 0) v1 -- 0x7f28a8104d40 con 0x7f28a80fe150 2024-04-02T04:56:21.735 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.731+0000 7f289e7fc640 1 -- 172.21.15.152:0/2346293981 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "osd pool get", "pool": "cephfs.cephfs.data", "var": "pg_num"}]=0 v46) v1 ==== 107+0+11 (secure 0 0 0) 0x7f2898078fa0 con 0x7f28a80fe150 2024-04-02T04:56:21.736 INFO:teuthology.orchestra.run.smithi152.stdout:pg_num: 32 2024-04-02T04:56:21.738 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.733+0000 7f28b0225640 1 -- 172.21.15.152:0/2346293981 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2894069210 msgr2=0x7f289406b6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:21.738 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.733+0000 7f28b0225640 1 --2- 172.21.15.152:0/2346293981 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2894069210 0x7f289406b6c0 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 crypto rx=0x7f28a810e1c0 tx=0x7f28a400b040 comp rx=0 tx=0).stop 2024-04-02T04:56:21.738 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.733+0000 7f28b0225640 1 -- 172.21.15.152:0/2346293981 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f28a80fe150 msgr2=0x7f28a810d240 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:21.738 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.733+0000 7f28b0225640 1 --2- 172.21.15.152:0/2346293981 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f28a80fe150 0x7f28a810d240 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 crypto rx=0x7f289802bda0 tx=0x7f2898009da0 comp rx=0 tx=0).stop 2024-04-02T04:56:21.738 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.733+0000 7f28b0225640 1 -- 172.21.15.152:0/2346293981 shutdown_connections 2024-04-02T04:56:21.738 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.733+0000 7f28b0225640 1 --2- 172.21.15.152:0/2346293981 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2894069210 0x7f289406b6c0 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:21.738 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.733+0000 7f28b0225640 1 --2- 172.21.15.152:0/2346293981 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f28a80fe150 0x7f28a810d240 unknown :-1 s=CLOSED pgs=259 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:21.738 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.733+0000 7f28b0225640 1 --2- 172.21.15.152:0/2346293981 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f28a80fd800 0x7f28a810cd00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:21.738 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.733+0000 7f28b0225640 1 -- 172.21.15.152:0/2346293981 >> 172.21.15.152:0/2346293981 conn(0x7f28a80f6000 msgr2=0x7f28a80f7690 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:21.738 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.733+0000 7f28b0225640 1 -- 172.21.15.152:0/2346293981 shutdown_connections 2024-04-02T04:56:21.738 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.733+0000 7f28b0225640 1 -- 172.21.15.152:0/2346293981 wait complete. 2024-04-02T04:56:21.748 DEBUG:teuthology.orchestra.run.smithi152:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs dump --format=json 2024-04-02T04:56:21.848 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.841+0000 7ff9854ae640 1 -- 172.21.15.152:0/242799118 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff9800fb0a0 msgr2=0x7ff9800fb4b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:21.848 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.841+0000 7ff9854ae640 1 --2- 172.21.15.152:0/242799118 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff9800fb0a0 0x7ff9800fb4b0 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 crypto rx=0x7ff9700048d0 tx=0x7ff970009da0 comp rx=0 tx=0).stop 2024-04-02T04:56:21.848 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.841+0000 7ff9854ae640 1 -- 172.21.15.152:0/242799118 shutdown_connections 2024-04-02T04:56:21.848 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.841+0000 7ff9854ae640 1 --2- 172.21.15.152:0/242799118 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff9800fc370 0x7ff9800fc7e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:21.848 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.841+0000 7ff9854ae640 1 --2- 172.21.15.152:0/242799118 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff9800fb0a0 0x7ff9800fb4b0 unknown :-1 s=CLOSED pgs=260 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:21.848 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.841+0000 7ff9854ae640 1 -- 172.21.15.152:0/242799118 >> 172.21.15.152:0/242799118 conn(0x7ff9800f6840 msgr2=0x7ff9800f8c90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:21.849 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.841+0000 7ff9854ae640 1 -- 172.21.15.152:0/242799118 shutdown_connections 2024-04-02T04:56:21.849 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.841+0000 7ff9854ae640 1 -- 172.21.15.152:0/242799118 wait complete. 2024-04-02T04:56:21.849 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.842+0000 7ff9854ae640 1 Processor -- start 2024-04-02T04:56:21.849 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.842+0000 7ff9854ae640 1 -- start start 2024-04-02T04:56:21.849 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.842+0000 7ff9854ae640 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff9800fb0a0 0x7ff980064dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:21.849 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.842+0000 7ff9854ae640 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff9800fc370 0x7ff980111ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:21.849 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.842+0000 7ff9854ae640 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff980111ff0 con 0x7ff9800fb0a0 2024-04-02T04:56:21.849 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.842+0000 7ff9854ae640 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff980112130 con 0x7ff9800fc370 2024-04-02T04:56:21.849 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.842+0000 7ff97effd640 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff9800fb0a0 0x7ff980064dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:21.849 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.842+0000 7ff97effd640 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff9800fb0a0 0x7ff980064dd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:37584/0 (socket says 172.21.15.152:37584) 2024-04-02T04:56:21.849 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.842+0000 7ff97effd640 1 -- 172.21.15.152:0/1016621657 learned_addr learned my addr 172.21.15.152:0/1016621657 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:56:21.849 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.842+0000 7ff97e7fc640 1 --2- 172.21.15.152:0/1016621657 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff9800fc370 0x7ff980111ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:21.849 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.842+0000 7ff97effd640 1 -- 172.21.15.152:0/1016621657 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff9800fc370 msgr2=0x7ff980111ab0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:21.849 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.842+0000 7ff97effd640 1 --2- 172.21.15.152:0/1016621657 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff9800fc370 0x7ff980111ab0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:21.850 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.842+0000 7ff97effd640 1 -- 172.21.15.152:0/1016621657 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff970005040 con 0x7ff9800fb0a0 2024-04-02T04:56:21.850 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.842+0000 7ff97e7fc640 1 --2- 172.21.15.152:0/1016621657 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff9800fc370 0x7ff980111ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).send_auth_request state changed! 2024-04-02T04:56:21.850 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.843+0000 7ff97effd640 1 --2- 172.21.15.152:0/1016621657 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff9800fb0a0 0x7ff980064dd0 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7ff9700048a0 tx=0x7ff970009da0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:21.850 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.843+0000 7ff9677fe640 1 -- 172.21.15.152:0/1016621657 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7ff9700094c0 con 0x7ff9800fb0a0 2024-04-02T04:56:21.850 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.843+0000 7ff9677fe640 1 -- 172.21.15.152:0/1016621657 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7ff970009620 con 0x7ff9800fb0a0 2024-04-02T04:56:21.850 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.843+0000 7ff9677fe640 1 -- 172.21.15.152:0/1016621657 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7ff97003b640 con 0x7ff9800fb0a0 2024-04-02T04:56:21.850 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.843+0000 7ff9854ae640 1 -- 172.21.15.152:0/1016621657 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff980112330 con 0x7ff9800fb0a0 2024-04-02T04:56:21.850 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.843+0000 7ff9854ae640 1 -- 172.21.15.152:0/1016621657 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff980112850 con 0x7ff9800fb0a0 2024-04-02T04:56:21.850 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.844+0000 7ff9854ae640 1 -- 172.21.15.152:0/1016621657 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff9801055a0 con 0x7ff9800fb0a0 2024-04-02T04:56:21.851 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.844+0000 7ff9677fe640 1 -- 172.21.15.152:0/1016621657 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7ff97002f070 con 0x7ff9800fb0a0 2024-04-02T04:56:21.851 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.844+0000 7ff9677fe640 1 --2- 172.21.15.152:0/1016621657 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7ff968069210 0x7ff96806b6c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:21.851 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.845+0000 7ff9677fe640 1 -- 172.21.15.152:0/1016621657 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7ff9700a97c0 con 0x7ff9800fb0a0 2024-04-02T04:56:21.851 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.845+0000 7ff97e7fc640 1 --2- 172.21.15.152:0/1016621657 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7ff968069210 0x7ff96806b6c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:21.851 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.845+0000 7ff97e7fc640 1 --2- 172.21.15.152:0/1016621657 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7ff968069210 0x7ff96806b6c0 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7ff980064b80 tx=0x7ff97400b040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:21.853 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:21.849+0000 7ff9677fe640 1 -- 172.21.15.152:0/1016621657 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7ff970075a20 con 0x7ff9800fb0a0 2024-04-02T04:56:22.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:21 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/2315409714' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T04:56:22.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:21 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/3414081345' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-02T04:56:22.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:21 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/4162559691' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs.cephfs.meta", "var": "pg_num"}]: dispatch 2024-04-02T04:56:22.007 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:21 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/2346293981' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs.cephfs.data", "var": "pg_num"}]: dispatch 2024-04-02T04:56:22.057 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:22.053+0000 7ff9854ae640 1 -- 172.21.15.152:0/1016621657 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "fs dump", "format": "json"} v 0) v1 -- 0x7ff980103c10 con 0x7ff9800fb0a0 2024-04-02T04:56:22.058 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:22.053+0000 7ff9677fe640 1 -- 172.21.15.152:0/1016621657 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump", "format": "json"}]=0 dumped fsmap epoch 20 v20) v1 ==== 94+0+4725 (secure 0 0 0) 0x7ff970079040 con 0x7ff9800fb0a0 2024-04-02T04:56:22.058 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:56:22.058 INFO:teuthology.orchestra.run.smithi152.stdout:{"epoch":20,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[{"mdsmap":{"epoch":20,"flags":50,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":true},"ever_allowed_features":32,"explicitly_allowed_features":32,"created":"2024-04-02T04:55:50.446442+0000","modified":"2024-04-02T04:56:08.533665+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":46,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":2,"in":[0,1],"up":{"mds_0":14456,"mds_1":14470},"failed":[],"damaged":[],"stopped":[],"info":{"gid_14456":{"gid":14456,"name":"cephfs.smithi152.lluqrs","rank":0,"incarnation":4,"state":"up:active","state_seq":5,"addr":"172.21.15.152:6827/4038990242","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6826","nonce":4038990242},{"type":"v1","addr":"172.21.15.152:6827","nonce":4038990242}]},"join_fscid":1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_14470":{"gid":14470,"name":"cephfs.smithi152.ywghzy","rank":1,"incarnation":15,"state":"up:active","state_seq":6,"addr":"172.21.15.152:6829/3554217123","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.152:6828","nonce":3554217123},{"type":"v1","addr":"172.21.15.152:6829","nonce":3554217123}]},"join_fscid":1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24297":{"gid":24297,"name":"cephfs.smithi154.rbkmwp","rank":0,"incarnation":0,"state":"up:standby-replay","state_seq":2,"addr":"172.21.15.154:6827/53707828","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6826","nonce":53707828},{"type":"v1","addr":"172.21.15.154:6827","nonce":53707828}]},"join_fscid":1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}},"gid_24301":{"gid":24301,"name":"cephfs.smithi154.llvaoh","rank":1,"incarnation":0,"state":"up:standby-replay","state_seq":1,"addr":"172.21.15.154:6825/1569612467","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.154:6824","nonce":1569612467},{"type":"v1","addr":"172.21.15.154:6825","nonce":1569612467}]},"join_fscid":1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":1},"id":1}]} 2024-04-02T04:56:22.059 INFO:teuthology.orchestra.run.smithi152.stderr:dumped fsmap epoch 20 2024-04-02T04:56:22.060 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:22.055+0000 7ff9854ae640 1 -- 172.21.15.152:0/1016621657 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7ff968069210 msgr2=0x7ff96806b6c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:22.060 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:22.055+0000 7ff9854ae640 1 --2- 172.21.15.152:0/1016621657 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7ff968069210 0x7ff96806b6c0 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 crypto rx=0x7ff980064b80 tx=0x7ff97400b040 comp rx=0 tx=0).stop 2024-04-02T04:56:22.060 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:22.055+0000 7ff9854ae640 1 -- 172.21.15.152:0/1016621657 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff9800fb0a0 msgr2=0x7ff980064dd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:22.060 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:22.055+0000 7ff9854ae640 1 --2- 172.21.15.152:0/1016621657 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff9800fb0a0 0x7ff980064dd0 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 crypto rx=0x7ff9700048a0 tx=0x7ff970009da0 comp rx=0 tx=0).stop 2024-04-02T04:56:22.060 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:22.055+0000 7ff9854ae640 1 -- 172.21.15.152:0/1016621657 shutdown_connections 2024-04-02T04:56:22.060 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:22.055+0000 7ff9854ae640 1 --2- 172.21.15.152:0/1016621657 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7ff968069210 0x7ff96806b6c0 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:22.060 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:22.055+0000 7ff9854ae640 1 --2- 172.21.15.152:0/1016621657 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7ff9800fc370 0x7ff980111ab0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:22.061 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:22.055+0000 7ff9854ae640 1 --2- 172.21.15.152:0/1016621657 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7ff9800fb0a0 0x7ff980064dd0 unknown :-1 s=CLOSED pgs=261 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:22.061 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:22.055+0000 7ff9854ae640 1 -- 172.21.15.152:0/1016621657 >> 172.21.15.152:0/1016621657 conn(0x7ff9800f6840 msgr2=0x7ff9800ff5a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:22.061 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:22.055+0000 7ff9854ae640 1 -- 172.21.15.152:0/1016621657 shutdown_connections 2024-04-02T04:56:22.061 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:22.055+0000 7ff9854ae640 1 -- 172.21.15.152:0/1016621657 wait complete. 2024-04-02T04:56:22.071 DEBUG:tasks.fs:fs fscid=1,name=cephfs state = {'epoch': 20, 'max_mds': 2, 'flags': 50} 2024-04-02T04:56:22.071 INFO:teuthology.run_tasks:Running task kclient... 2024-04-02T04:56:22.149 INFO:tasks.kclient:Mounting kernel clients... 2024-04-02T04:56:22.150 INFO:tasks.kclient:config is {'client.0': {}, 'client.1': {}} 2024-04-02T04:56:22.150 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-02T04:56:22.150 DEBUG:teuthology.orchestra.run.smithi152:> ip netns list 2024-04-02T04:56:22.165 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-02T04:56:22.166 DEBUG:teuthology.orchestra.run.smithi152:> sudo ip link delete ceph-brx 2024-04-02T04:56:22.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:21 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/2315409714' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-02T04:56:22.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:21 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/3414081345' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-02T04:56:22.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:21 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/4162559691' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs.cephfs.meta", "var": "pg_num"}]: dispatch 2024-04-02T04:56:22.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:21 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/2346293981' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": "cephfs.cephfs.data", "var": "pg_num"}]: dispatch 2024-04-02T04:56:22.235 INFO:teuthology.orchestra.run.smithi152.stderr:Cannot find device "ceph-brx" 2024-04-02T04:56:22.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T04:56:22.236 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-02T04:56:22.236 DEBUG:teuthology.orchestra.run.smithi154:> ip netns list 2024-04-02T04:56:22.256 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-02T04:56:22.256 DEBUG:teuthology.orchestra.run.smithi154:> sudo ip link delete ceph-brx 2024-04-02T04:56:22.332 INFO:teuthology.orchestra.run.smithi154.stderr:Cannot find device "ceph-brx" 2024-04-02T04:56:22.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-02T04:56:22.337 INFO:tasks.kclient:client.0 config is {'syntax': 'v1'} 2024-04-02T04:56:22.337 INFO:tasks.cephfs.mount:cephfs_mntpt = None 2024-04-02T04:56:22.337 INFO:tasks.cephfs.mount:hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-04-02T04:56:22.338 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph fs ls 2024-04-02T04:56:22.598 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:56:22.979 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:22 smithi152 ceph-mon[29073]: pgmap v190: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 4.9 KiB/s rd, 0 B/s wr, 6 op/s 2024-04-02T04:56:22.979 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:22 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/1016621657' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-02T04:56:23.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:22 smithi154 ceph-mon[35729]: pgmap v190: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 4.9 KiB/s rd, 0 B/s wr, 6 op/s 2024-04-02T04:56:23.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:22 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/1016621657' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-04-02T04:56:24.262 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.256+0000 7f06483d7700 1 -- 172.21.15.152:0/3347889768 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f06401037b0 msgr2=0x7f06400fce20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:24.263 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.256+0000 7f06483d7700 1 --2- 172.21.15.152:0/3347889768 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f06401037b0 0x7f06400fce20 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 crypto rx=0x7f063c004940 tx=0x7f063c009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:24.263 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.256+0000 7f06483d7700 1 -- 172.21.15.152:0/3347889768 shutdown_connections 2024-04-02T04:56:24.263 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.256+0000 7f06483d7700 1 --2- 172.21.15.152:0/3347889768 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f06401037b0 0x7f06400fce20 unknown :-1 s=CLOSED pgs=262 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:24.263 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.256+0000 7f06483d7700 1 --2- 172.21.15.152:0/3347889768 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0640102e10 0x7f06401031e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:24.263 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.256+0000 7f06483d7700 1 -- 172.21.15.152:0/3347889768 >> 172.21.15.152:0/3347889768 conn(0x7f06400f89a0 msgr2=0x7f06400fadb0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:24.263 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.256+0000 7f06483d7700 1 -- 172.21.15.152:0/3347889768 shutdown_connections 2024-04-02T04:56:24.263 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.256+0000 7f06483d7700 1 -- 172.21.15.152:0/3347889768 wait complete. 2024-04-02T04:56:24.263 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.256+0000 7f06483d7700 1 Processor -- start 2024-04-02T04:56:24.263 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.257+0000 7f06483d7700 1 -- start start 2024-04-02T04:56:24.263 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.257+0000 7f06483d7700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0640102e10 0x7f0640113ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:24.263 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.257+0000 7f06483d7700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f06401037b0 0x7f06401140e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:24.263 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.257+0000 7f06483d7700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0640114730 con 0x7f0640102e10 2024-04-02T04:56:24.263 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.257+0000 7f06483d7700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f064010db20 con 0x7f06401037b0 2024-04-02T04:56:24.264 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.257+0000 7f0646173700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0640102e10 0x7f0640113ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:24.264 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.257+0000 7f0646173700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0640102e10 0x7f0640113ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:37608/0 (socket says 172.21.15.152:37608) 2024-04-02T04:56:24.264 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.257+0000 7f0646173700 1 -- 172.21.15.152:0/3197635826 learned_addr learned my addr 172.21.15.152:0/3197635826 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:56:24.264 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.257+0000 7f0645972700 1 --2- 172.21.15.152:0/3197635826 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f06401037b0 0x7f06401140e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:24.264 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.257+0000 7f0646173700 1 -- 172.21.15.152:0/3197635826 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f06401037b0 msgr2=0x7f06401140e0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:24.264 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.257+0000 7f0646173700 1 --2- 172.21.15.152:0/3197635826 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f06401037b0 0x7f06401140e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:24.264 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.257+0000 7f0646173700 1 -- 172.21.15.152:0/3197635826 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f063c005040 con 0x7f0640102e10 2024-04-02T04:56:24.264 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.257+0000 7f0646173700 1 --2- 172.21.15.152:0/3197635826 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0640102e10 0x7f0640113ba0 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7f063000c8c0 tx=0x7f06300044c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:24.264 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.257+0000 7f0636ffd700 1 -- 172.21.15.152:0/3197635826 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f063000a980 con 0x7f0640102e10 2024-04-02T04:56:24.264 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.257+0000 7f0636ffd700 1 -- 172.21.15.152:0/3197635826 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f063000ce60 con 0x7f0640102e10 2024-04-02T04:56:24.264 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.257+0000 7f06483d7700 1 -- 172.21.15.152:0/3197635826 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f064010de00 con 0x7f0640102e10 2024-04-02T04:56:24.264 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.257+0000 7f0636ffd700 1 -- 172.21.15.152:0/3197635826 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f06300115e0 con 0x7f0640102e10 2024-04-02T04:56:24.264 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.258+0000 7f06483d7700 1 -- 172.21.15.152:0/3197635826 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f064010e3c0 con 0x7f0640102e10 2024-04-02T04:56:24.265 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.258+0000 7f06483d7700 1 -- 172.21.15.152:0/3197635826 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f06401052e0 con 0x7f0640102e10 2024-04-02T04:56:24.265 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.259+0000 7f0636ffd700 1 -- 172.21.15.152:0/3197635826 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f0630012400 con 0x7f0640102e10 2024-04-02T04:56:24.265 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.259+0000 7f0636ffd700 1 --2- 172.21.15.152:0/3197635826 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f062c0692a0 0x7f062c06b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:24.265 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.259+0000 7f0636ffd700 1 -- 172.21.15.152:0/3197635826 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7f06300872e0 con 0x7f0640102e10 2024-04-02T04:56:24.265 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.259+0000 7f0645972700 1 --2- 172.21.15.152:0/3197635826 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f062c0692a0 0x7f062c06b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:24.266 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.260+0000 7f0645972700 1 --2- 172.21.15.152:0/3197635826 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f062c0692a0 0x7f062c06b750 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f063c00b3b0 tx=0x7f063c009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:24.268 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.264+0000 7f0636ffd700 1 -- 172.21.15.152:0/3197635826 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f0630057050 con 0x7f0640102e10 2024-04-02T04:56:24.498 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.493+0000 7f06483d7700 1 -- 172.21.15.152:0/3197635826 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7f064010f1f0 con 0x7f0640102e10 2024-04-02T04:56:24.498 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.493+0000 7f0636ffd700 1 -- 172.21.15.152:0/3197635826 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v20) v1 ==== 53+0+83 (secure 0 0 0) 0x7f06300126b0 con 0x7f0640102e10 2024-04-02T04:56:24.498 INFO:teuthology.orchestra.run.smithi152.stdout:name: cephfs, metadata pool: cephfs.cephfs.meta, data pools: [cephfs.cephfs.data ] 2024-04-02T04:56:24.500 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.495+0000 7f06483d7700 1 -- 172.21.15.152:0/3197635826 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f062c0692a0 msgr2=0x7f062c06b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:24.500 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.495+0000 7f06483d7700 1 --2- 172.21.15.152:0/3197635826 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f062c0692a0 0x7f062c06b750 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 crypto rx=0x7f063c00b3b0 tx=0x7f063c009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:24.500 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.495+0000 7f06483d7700 1 -- 172.21.15.152:0/3197635826 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0640102e10 msgr2=0x7f0640113ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:24.500 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.495+0000 7f06483d7700 1 --2- 172.21.15.152:0/3197635826 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0640102e10 0x7f0640113ba0 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 crypto rx=0x7f063000c8c0 tx=0x7f06300044c0 comp rx=0 tx=0).stop 2024-04-02T04:56:24.500 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.495+0000 7f06483d7700 1 -- 172.21.15.152:0/3197635826 shutdown_connections 2024-04-02T04:56:24.500 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.495+0000 7f06483d7700 1 --2- 172.21.15.152:0/3197635826 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f062c0692a0 0x7f062c06b750 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:24.500 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.495+0000 7f06483d7700 1 --2- 172.21.15.152:0/3197635826 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0640102e10 0x7f0640113ba0 unknown :-1 s=CLOSED pgs=263 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:24.500 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.495+0000 7f06483d7700 1 --2- 172.21.15.152:0/3197635826 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f06401037b0 0x7f06401140e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:24.501 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.495+0000 7f06483d7700 1 -- 172.21.15.152:0/3197635826 >> 172.21.15.152:0/3197635826 conn(0x7f06400f89a0 msgr2=0x7f0640102220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:24.501 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.495+0000 7f06483d7700 1 -- 172.21.15.152:0/3197635826 shutdown_connections 2024-04-02T04:56:24.501 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:24.495+0000 7f06483d7700 1 -- 172.21.15.152:0/3197635826 wait complete. 2024-04-02T04:56:25.074 INFO:tasks.cephfs.mount:Mounting default Ceph FS; just confirmed its presence on cluster 2024-04-02T04:56:25.075 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-04-02T04:56:25.075 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi152.front.sepia.ceph.com 2024-04-02T04:56:25.075 INFO:tasks.cephfs.mount:self.client.name = client.0 2024-04-02T04:56:25.075 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.0 2024-04-02T04:56:25.075 INFO:tasks.cephfs.mount:self.cephfs_name = None 2024-04-02T04:56:25.075 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2024-04-02T04:56:25.075 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-04-02T04:56:25.075 INFO:tasks.cephfs.mount:Setting the 'None' netns for '/home/ubuntu/cephtest/mnt.0' 2024-04-02T04:56:25.075 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-02T04:56:25.075 DEBUG:teuthology.orchestra.run.smithi152:> ip addr 2024-04-02T04:56:25.091 INFO:teuthology.orchestra.run.smithi152.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-04-02T04:56:25.091 INFO:teuthology.orchestra.run.smithi152.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-04-02T04:56:25.091 INFO:teuthology.orchestra.run.smithi152.stdout: inet 127.0.0.1/8 scope host lo 2024-04-02T04:56:25.091 INFO:teuthology.orchestra.run.smithi152.stdout: valid_lft forever preferred_lft forever 2024-04-02T04:56:25.091 INFO:teuthology.orchestra.run.smithi152.stdout: inet6 ::1/128 scope host 2024-04-02T04:56:25.091 INFO:teuthology.orchestra.run.smithi152.stdout: valid_lft forever preferred_lft forever 2024-04-02T04:56:25.091 INFO:teuthology.orchestra.run.smithi152.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-04-02T04:56:25.092 INFO:teuthology.orchestra.run.smithi152.stdout: link/ether 0c:c4:7a:da:6c:44 brd ff:ff:ff:ff:ff:ff 2024-04-02T04:56:25.092 INFO:teuthology.orchestra.run.smithi152.stdout: altname enp7s0f0 2024-04-02T04:56:25.092 INFO:teuthology.orchestra.run.smithi152.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-04-02T04:56:25.092 INFO:teuthology.orchestra.run.smithi152.stdout: link/ether 0c:c4:7a:da:6c:45 brd ff:ff:ff:ff:ff:ff 2024-04-02T04:56:25.092 INFO:teuthology.orchestra.run.smithi152.stdout: altname enp7s0f1 2024-04-02T04:56:25.092 INFO:teuthology.orchestra.run.smithi152.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-04-02T04:56:25.092 INFO:teuthology.orchestra.run.smithi152.stdout: link/ether 0c:c4:7a:88:6d:e2 brd ff:ff:ff:ff:ff:ff 2024-04-02T04:56:25.092 INFO:teuthology.orchestra.run.smithi152.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-04-02T04:56:25.092 INFO:teuthology.orchestra.run.smithi152.stdout: link/ether 0c:c4:7a:88:6d:e3 brd ff:ff:ff:ff:ff:ff 2024-04-02T04:56:25.092 INFO:teuthology.orchestra.run.smithi152.stdout: inet 172.21.15.152/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-04-02T04:56:25.092 INFO:teuthology.orchestra.run.smithi152.stdout: valid_lft 42265sec preferred_lft 42265sec 2024-04-02T04:56:25.092 INFO:teuthology.orchestra.run.smithi152.stdout: inet6 fe80::ec4:7aff:fe88:6de3/64 scope link 2024-04-02T04:56:25.092 INFO:teuthology.orchestra.run.smithi152.stdout: valid_lft forever preferred_lft forever 2024-04-02T04:56:25.093 INFO:tasks.cephfs.mount:Setuping the 'ceph-brx' with 192.168.255.254/16 2024-04-02T04:56:25.093 DEBUG:teuthology.orchestra.run.smithi152:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-04-02T04:56:25.093 DEBUG:teuthology.orchestra.run.smithi152:> set -e 2024-04-02T04:56:25.093 DEBUG:teuthology.orchestra.run.smithi152:> sudo ip link add name ceph-brx type bridge 2024-04-02T04:56:25.093 DEBUG:teuthology.orchestra.run.smithi152:> sudo ip addr flush dev ceph-brx 2024-04-02T04:56:25.093 DEBUG:teuthology.orchestra.run.smithi152:> sudo ip link set ceph-brx up 2024-04-02T04:56:25.093 DEBUG:teuthology.orchestra.run.smithi152:> sudo ip addr add 192.168.255.254/16 brd 192.168.255.255 dev ceph-brx 2024-04-02T04:56:25.093 DEBUG:teuthology.orchestra.run.smithi152:> ') 2024-04-02T04:56:25.180 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:25 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-04-02T04:56:25.299 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:25 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-02T04:56:25.300 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:25 smithi152 ceph-mon[29073]: pgmap v191: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 5.2 KiB/s rd, 7 op/s 2024-04-02T04:56:25.300 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:25 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/3197635826' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2024-04-02T04:56:25.300 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:25 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:56:25.306 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-02T04:56:25.306 DEBUG:teuthology.orchestra.run.smithi152:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-04-02T04:56:25.335 INFO:teuthology.orchestra.run.smithi152.stdout:1 2024-04-02T04:56:25.337 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-02T04:56:25.337 DEBUG:teuthology.orchestra.run.smithi152:> route 2024-04-02T04:56:25.404 INFO:teuthology.orchestra.run.smithi152.stdout:Kernel IP routing table 2024-04-02T04:56:25.404 INFO:teuthology.orchestra.run.smithi152.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-04-02T04:56:25.404 INFO:teuthology.orchestra.run.smithi152.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-04-02T04:56:25.404 INFO:teuthology.orchestra.run.smithi152.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-04-02T04:56:25.405 INFO:teuthology.orchestra.run.smithi152.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-04-02T04:56:25.405 DEBUG:teuthology.orchestra.run.smithi152:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-04-02T04:56:25.406 DEBUG:teuthology.orchestra.run.smithi152:> set -e 2024-04-02T04:56:25.406 DEBUG:teuthology.orchestra.run.smithi152:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-04-02T04:56:25.406 DEBUG:teuthology.orchestra.run.smithi152:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-04-02T04:56:25.406 DEBUG:teuthology.orchestra.run.smithi152:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-04-02T04:56:25.406 DEBUG:teuthology.orchestra.run.smithi152:> ') 2024-04-02T04:56:25.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:25 smithi154 ceph-mon[35729]: pgmap v191: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 5.2 KiB/s rd, 7 op/s 2024-04-02T04:56:25.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:25 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/3197635826' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2024-04-02T04:56:25.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:25 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:56:25.493 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:25 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-04-02T04:56:25.708 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:25 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-02T04:56:25.714 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-02T04:56:25.714 DEBUG:teuthology.orchestra.run.smithi152:> ip netns list 2024-04-02T04:56:25.732 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-02T04:56:25.732 DEBUG:teuthology.orchestra.run.smithi152:> ip netns list-id 2024-04-02T04:56:25.788 DEBUG:teuthology.orchestra.run.smithi152:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-04-02T04:56:25.788 DEBUG:teuthology.orchestra.run.smithi152:> set -e 2024-04-02T04:56:25.788 DEBUG:teuthology.orchestra.run.smithi152:> sudo ip netns add ceph-ns--home-ubuntu-cephtest-mnt.0 2024-04-02T04:56:25.788 DEBUG:teuthology.orchestra.run.smithi152:> sudo ip netns set ceph-ns--home-ubuntu-cephtest-mnt.0 0 2024-04-02T04:56:25.788 DEBUG:teuthology.orchestra.run.smithi152:> ') 2024-04-02T04:56:25.875 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:25 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-04-02T04:56:25.903 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:25 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-02T04:56:25.910 INFO:tasks.cephfs.mount:Setuping the netns 'ceph-ns--home-ubuntu-cephtest-mnt.0' with 192.168.0.1/16 2024-04-02T04:56:25.910 DEBUG:teuthology.orchestra.run.smithi152:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-04-02T04:56:25.910 DEBUG:teuthology.orchestra.run.smithi152:> set -e 2024-04-02T04:56:25.910 DEBUG:teuthology.orchestra.run.smithi152:> sudo ip link add veth0 netns ceph-ns--home-ubuntu-cephtest-mnt.0 type veth peer name brx.0 2024-04-02T04:56:25.910 DEBUG:teuthology.orchestra.run.smithi152:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip addr add 192.168.0.1/16 brd 192.168.255.255 dev veth0 2024-04-02T04:56:25.910 DEBUG:teuthology.orchestra.run.smithi152:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip link set veth0 up 2024-04-02T04:56:25.910 DEBUG:teuthology.orchestra.run.smithi152:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip link set lo up 2024-04-02T04:56:25.910 DEBUG:teuthology.orchestra.run.smithi152:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.0 ip route add default via 192.168.255.254 2024-04-02T04:56:25.910 DEBUG:teuthology.orchestra.run.smithi152:> ') 2024-04-02T04:56:25.958 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:25 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-04-02T04:56:26.045 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:26 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-02T04:56:26.054 DEBUG:teuthology.orchestra.run.smithi152:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-04-02T04:56:26.054 DEBUG:teuthology.orchestra.run.smithi152:> set -e 2024-04-02T04:56:26.054 DEBUG:teuthology.orchestra.run.smithi152:> sudo ip link set brx.0 up 2024-04-02T04:56:26.054 DEBUG:teuthology.orchestra.run.smithi152:> sudo ip link set dev brx.0 master ceph-brx 2024-04-02T04:56:26.054 DEBUG:teuthology.orchestra.run.smithi152:> ') 2024-04-02T04:56:26.102 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:26 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-04-02T04:56:26.130 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:26 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-02T04:56:26.136 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-04-02T04:56:26.136 DEBUG:teuthology.orchestra.run.smithi152:> mkdir -p -v /home/ubuntu/cephtest/mnt.0 2024-04-02T04:56:26.150 INFO:teuthology.orchestra.run.smithi152.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.0' 2024-04-02T04:56:26.150 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-04-02T04:56:26.150 DEBUG:teuthology.orchestra.run.smithi152:> chmod 0000 /home/ubuntu/cephtest/mnt.0 2024-04-02T04:56:26.205 INFO:tasks.cephfs.kernel_mount:mounting using device: :/ 2024-04-02T04:56:26.205 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-02T04:56:26.205 DEBUG:teuthology.orchestra.run.smithi152:> sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/mount -t ceph :/ /home/ubuntu/cephtest/mnt.0 -v -o norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,name=0,mds_namespace=cephfs 2024-04-02T04:56:26.296 INFO:teuthology.orchestra.run.smithi152.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,name=0,mds_namespace=cephfs 2024-04-02T04:56:26.296 INFO:teuthology.orchestra.run.smithi152.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,mds_namespace=cephfs". 2024-04-02T04:56:26.296 INFO:teuthology.orchestra.run.smithi152.stdout:invalid new device string format 2024-04-02T04:56:26.416 INFO:teuthology.orchestra.run.smithi152.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,name=0,mds_namespace=cephfs 2024-04-02T04:56:26.416 INFO:teuthology.orchestra.run.smithi152.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,mds_namespace=cephfs". 2024-04-02T04:56:26.417 INFO:teuthology.orchestra.run.smithi152.stdout:invalid new device string format 2024-04-02T04:56:26.417 INFO:teuthology.orchestra.run.smithi152.stdout:mount.ceph: resolved to: "172.21.15.152:3300,172.21.15.154:3300" 2024-04-02T04:56:26.417 INFO:teuthology.orchestra.run.smithi152.stdout:mount.ceph: switching to using v1 address with old syntax 2024-04-02T04:56:26.421 INFO:teuthology.orchestra.run.smithi152.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,name=0,mds_namespace=cephfs 2024-04-02T04:56:26.421 INFO:teuthology.orchestra.run.smithi152.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,mds_namespace=cephfs". 2024-04-02T04:56:26.421 INFO:teuthology.orchestra.run.smithi152.stdout:invalid new device string format 2024-04-02T04:56:26.421 INFO:teuthology.orchestra.run.smithi152.stdout:mount.ceph: resolved to: "172.21.15.152:3300,172.21.15.154:3300" 2024-04-02T04:56:26.421 INFO:teuthology.orchestra.run.smithi152.stdout:mount.ceph: switching to using v1 address with old syntax 2024-04-02T04:56:26.422 INFO:teuthology.orchestra.run.smithi152.stdout:mount.ceph: resolved to: "172.21.15.152:6789,172.21.15.154:6789" 2024-04-02T04:56:26.422 INFO:teuthology.orchestra.run.smithi152.stdout:mount.ceph: trying mount with old device syntax: 172.21.15.152:6789,172.21.15.154:6789:/ 2024-04-02T04:56:26.422 INFO:teuthology.orchestra.run.smithi152.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=0,mds_namespace=cephfs,key=0,fsid=2e0055d0-f0ac-11ee-b647-cb9ed24678a4" will pass to kernel 2024-04-02T04:56:26.422 INFO:teuthology.orchestra.run.smithi152.stdout:mount: /home/ubuntu/cephtest/mnt.0 does not contain SELinux labels. 2024-04-02T04:56:26.422 INFO:teuthology.orchestra.run.smithi152.stdout: You just mounted a file system that supports labels which does not 2024-04-02T04:56:26.422 INFO:teuthology.orchestra.run.smithi152.stdout: contain labels, onto an SELinux box. It is likely that confined 2024-04-02T04:56:26.423 INFO:teuthology.orchestra.run.smithi152.stdout: applications will generate AVC messages and not be allowed access to 2024-04-02T04:56:26.423 INFO:teuthology.orchestra.run.smithi152.stdout: this file system. For more details see restorecon(8) and mount(8). 2024-04-02T04:56:26.424 INFO:tasks.cephfs.kernel_mount:mount command passed 2024-04-02T04:56:26.424 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-02T04:56:26.424 DEBUG:teuthology.orchestra.run.smithi152:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.0 2024-04-02T04:56:26.454 DEBUG:teuthology.orchestra.run.smithi152:> (cd /home/ubuntu/cephtest/mnt.0 && exec stdin-killer --timeout=900 -- bash -c 'getfattr --only-values -n ceph.client_id .') 2024-04-02T04:56:26.544 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:26 stdin-killer INFO: expiration expected; waiting 900 seconds for command to complete 2024-04-02T04:56:26.552 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:26 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-02T04:56:26.558 INFO:teuthology.orchestra.run.smithi152.stdout:client24333 2024-04-02T04:56:26.558 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-02T04:56:26.558 DEBUG:teuthology.orchestra.run.smithi152:> dd if=/proc/self/mounts of=/dev/stdout 2024-04-02T04:56:26.575 DEBUG:teuthology.orchestra.run.smithi152:> (cd /home/ubuntu/cephtest/mnt.0 && exec stdin-killer --timeout=900 -- bash -c 'getfattr --only-values -n ceph.client_id .') 2024-04-02T04:56:26.665 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:26 stdin-killer INFO: expiration expected; waiting 900 seconds for command to complete 2024-04-02T04:56:26.672 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:26 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-02T04:56:26.678 INFO:teuthology.orchestra.run.smithi152.stdout:client24333 2024-04-02T04:56:26.679 DEBUG:teuthology.orchestra.run.smithi152:> (cd /home/ubuntu/cephtest/mnt.0 && exec stdin-killer --timeout=900 -- bash -c 'getfattr --only-values -n ceph.client_id .') 2024-04-02T04:56:26.729 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:26 stdin-killer INFO: expiration expected; waiting 900 seconds for command to complete 2024-04-02T04:56:26.736 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:26 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-02T04:56:26.742 INFO:teuthology.orchestra.run.smithi152.stdout:client24333 2024-04-02T04:56:26.742 DEBUG:teuthology.orchestra.run.smithi152:> (cd /home/ubuntu/cephtest/mnt.0 && exec stdin-killer --timeout=300 -- bash -c 'sudo dd if=/sys/kernel/debug/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4.client24333/status') 2024-04-02T04:56:26.792 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:26 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-04-02T04:56:26.807 INFO:teuthology.orchestra.run.smithi152.stdout:instance: client.24333 (3)172.21.15.152:0/469913195 2024-04-02T04:56:26.807 INFO:teuthology.orchestra.run.smithi152.stdout:blocklisted: false 2024-04-02T04:56:26.807 INFO:teuthology.orchestra.run.smithi152.stderr:0+1 records in 2024-04-02T04:56:26.807 INFO:teuthology.orchestra.run.smithi152.stderr:0+1 records out 2024-04-02T04:56:26.807 INFO:teuthology.orchestra.run.smithi152.stderr:71 bytes copied, 2.2625e-05 s, 3.1 MB/s 2024-04-02T04:56:26.808 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:26 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-02T04:56:26.814 INFO:tasks.kclient:client.1 config is {'syntax': 'v1'} 2024-04-02T04:56:26.814 INFO:tasks.cephfs.mount:cephfs_mntpt = None 2024-04-02T04:56:26.815 INFO:tasks.cephfs.mount:hostfs_mntpt = /home/ubuntu/cephtest/mnt.1 2024-04-02T04:56:26.815 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -- ceph fs ls 2024-04-02T04:56:27.067 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:27 smithi152 ceph-mon[29073]: pgmap v192: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 3 op/s 2024-04-02T04:56:27.073 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:56:27.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:27 smithi154 ceph-mon[35729]: pgmap v192: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 3 op/s 2024-04-02T04:56:28.664 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.651+0000 7f4e554c2700 1 -- 172.21.15.152:0/1354575309 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4e501026c0 msgr2=0x7f4e500fbd30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:28.664 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.651+0000 7f4e554c2700 1 --2- 172.21.15.152:0/1354575309 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4e501026c0 0x7f4e500fbd30 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 crypto rx=0x7f4e38004940 tx=0x7f4e38009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:28.664 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.651+0000 7f4e554c2700 1 -- 172.21.15.152:0/1354575309 shutdown_connections 2024-04-02T04:56:28.664 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.651+0000 7f4e554c2700 1 --2- 172.21.15.152:0/1354575309 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4e501026c0 0x7f4e500fbd30 unknown :-1 s=CLOSED pgs=264 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:28.664 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.651+0000 7f4e554c2700 1 --2- 172.21.15.152:0/1354575309 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4e50101d20 0x7f4e501020f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:28.664 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.651+0000 7f4e554c2700 1 -- 172.21.15.152:0/1354575309 >> 172.21.15.152:0/1354575309 conn(0x7f4e500f7870 msgr2=0x7f4e500f9c80 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:28.664 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.651+0000 7f4e554c2700 1 -- 172.21.15.152:0/1354575309 shutdown_connections 2024-04-02T04:56:28.664 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.651+0000 7f4e554c2700 1 -- 172.21.15.152:0/1354575309 wait complete. 2024-04-02T04:56:28.664 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.652+0000 7f4e554c2700 1 Processor -- start 2024-04-02T04:56:28.664 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.652+0000 7f4e554c2700 1 -- start start 2024-04-02T04:56:28.664 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.652+0000 7f4e554c2700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4e50101d20 0x7f4e5010eb40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:28.664 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.652+0000 7f4e554c2700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4e501026c0 0x7f4e5010f080 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:28.665 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.652+0000 7f4e554c2700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4e5010f5c0 con 0x7f4e501026c0 2024-04-02T04:56:28.665 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.652+0000 7f4e554c2700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4e5010f700 con 0x7f4e50101d20 2024-04-02T04:56:28.665 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.652+0000 7f4e4e7fc700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4e501026c0 0x7f4e5010f080 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:28.665 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.652+0000 7f4e4e7fc700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4e501026c0 0x7f4e5010f080 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:37630/0 (socket says 172.21.15.152:37630) 2024-04-02T04:56:28.665 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.652+0000 7f4e4e7fc700 1 -- 172.21.15.152:0/1036640068 learned_addr learned my addr 172.21.15.152:0/1036640068 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:56:28.665 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.653+0000 7f4e4effd700 1 --2- 172.21.15.152:0/1036640068 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4e50101d20 0x7f4e5010eb40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:28.665 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.653+0000 7f4e4e7fc700 1 -- 172.21.15.152:0/1036640068 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4e50101d20 msgr2=0x7f4e5010eb40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:28.665 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.653+0000 7f4e4e7fc700 1 --2- 172.21.15.152:0/1036640068 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4e50101d20 0x7f4e5010eb40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:28.665 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.653+0000 7f4e4e7fc700 1 -- 172.21.15.152:0/1036640068 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4e38005040 con 0x7f4e501026c0 2024-04-02T04:56:28.665 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.653+0000 7f4e4effd700 1 --2- 172.21.15.152:0/1036640068 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4e50101d20 0x7f4e5010eb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-02T04:56:28.665 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.653+0000 7f4e4e7fc700 1 --2- 172.21.15.152:0/1036640068 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4e501026c0 0x7f4e5010f080 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7f4e38009100 tx=0x7f4e38009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:28.665 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.653+0000 7f4e477fe700 1 -- 172.21.15.152:0/1036640068 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f4e3800e040 con 0x7f4e501026c0 2024-04-02T04:56:28.665 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.653+0000 7f4e477fe700 1 -- 172.21.15.152:0/1036640068 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f4e38013420 con 0x7f4e501026c0 2024-04-02T04:56:28.665 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.653+0000 7f4e554c2700 1 -- 172.21.15.152:0/1036640068 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4e5010f930 con 0x7f4e501026c0 2024-04-02T04:56:28.666 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.653+0000 7f4e477fe700 1 -- 172.21.15.152:0/1036640068 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f4e3801c610 con 0x7f4e501026c0 2024-04-02T04:56:28.666 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.653+0000 7f4e554c2700 1 -- 172.21.15.152:0/1036640068 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4e50117f50 con 0x7f4e501026c0 2024-04-02T04:56:28.666 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.654+0000 7f4e554c2700 1 -- 172.21.15.152:0/1036640068 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4e500fd470 con 0x7f4e501026c0 2024-04-02T04:56:28.666 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.655+0000 7f4e477fe700 1 -- 172.21.15.152:0/1036640068 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f4e3800baf0 con 0x7f4e501026c0 2024-04-02T04:56:28.666 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.655+0000 7f4e477fe700 1 --2- 172.21.15.152:0/1036640068 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f4e3c0692a0 0x7f4e3c06b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:28.666 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.655+0000 7f4e477fe700 1 -- 172.21.15.152:0/1036640068 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7f4e38088360 con 0x7f4e501026c0 2024-04-02T04:56:28.666 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.655+0000 7f4e4effd700 1 --2- 172.21.15.152:0/1036640068 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f4e3c0692a0 0x7f4e3c06b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:28.666 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.655+0000 7f4e4effd700 1 --2- 172.21.15.152:0/1036640068 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f4e3c0692a0 0x7f4e3c06b750 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f4e48001250 tx=0x7f4e48009610 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:28.666 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.660+0000 7f4e477fe700 1 -- 172.21.15.152:0/1036640068 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f4e380546e0 con 0x7f4e501026c0 2024-04-02T04:56:28.894 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.889+0000 7f4e554c2700 1 -- 172.21.15.152:0/1036640068 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "fs ls"} v 0) v1 -- 0x7f4e50108e70 con 0x7f4e501026c0 2024-04-02T04:56:28.894 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.890+0000 7f4e477fe700 1 -- 172.21.15.152:0/1036640068 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "fs ls"}]=0 v20) v1 ==== 53+0+83 (secure 0 0 0) 0x7f4e38057d00 con 0x7f4e501026c0 2024-04-02T04:56:28.894 INFO:teuthology.orchestra.run.smithi152.stdout:name: cephfs, metadata pool: cephfs.cephfs.meta, data pools: [cephfs.cephfs.data ] 2024-04-02T04:56:28.896 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.892+0000 7f4e554c2700 1 -- 172.21.15.152:0/1036640068 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f4e3c0692a0 msgr2=0x7f4e3c06b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:28.896 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.892+0000 7f4e554c2700 1 --2- 172.21.15.152:0/1036640068 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f4e3c0692a0 0x7f4e3c06b750 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 crypto rx=0x7f4e48001250 tx=0x7f4e48009610 comp rx=0 tx=0).stop 2024-04-02T04:56:28.896 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.892+0000 7f4e554c2700 1 -- 172.21.15.152:0/1036640068 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4e501026c0 msgr2=0x7f4e5010f080 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:28.896 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.892+0000 7f4e554c2700 1 --2- 172.21.15.152:0/1036640068 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4e501026c0 0x7f4e5010f080 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 crypto rx=0x7f4e38009100 tx=0x7f4e38009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:28.896 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.892+0000 7f4e554c2700 1 -- 172.21.15.152:0/1036640068 shutdown_connections 2024-04-02T04:56:28.896 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.892+0000 7f4e554c2700 1 --2- 172.21.15.152:0/1036640068 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f4e3c0692a0 0x7f4e3c06b750 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:28.896 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.892+0000 7f4e554c2700 1 --2- 172.21.15.152:0/1036640068 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f4e50101d20 0x7f4e5010eb40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:28.896 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.892+0000 7f4e554c2700 1 --2- 172.21.15.152:0/1036640068 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f4e501026c0 0x7f4e5010f080 unknown :-1 s=CLOSED pgs=265 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:28.896 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.892+0000 7f4e554c2700 1 -- 172.21.15.152:0/1036640068 >> 172.21.15.152:0/1036640068 conn(0x7f4e500f7870 msgr2=0x7f4e500f87a0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:28.896 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.892+0000 7f4e554c2700 1 -- 172.21.15.152:0/1036640068 shutdown_connections 2024-04-02T04:56:28.896 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:28.892+0000 7f4e554c2700 1 -- 172.21.15.152:0/1036640068 wait complete. 2024-04-02T04:56:29.251 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:29 smithi152 ceph-mon[29073]: pgmap v193: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 85 B/s wr, 3 op/s 2024-04-02T04:56:29.251 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:29 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/1036640068' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2024-04-02T04:56:29.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:29 smithi154 ceph-mon[35729]: pgmap v193: 65 pgs: 65 active+clean; 580 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 85 B/s wr, 3 op/s 2024-04-02T04:56:29.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:29 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/1036640068' entity='client.admin' cmd=[{"prefix": "fs ls"}]: dispatch 2024-04-02T04:56:29.593 INFO:tasks.cephfs.mount:Mounting default Ceph FS; just confirmed its presence on cluster 2024-04-02T04:56:29.593 INFO:tasks.cephfs.mount:Mounting Ceph FS. Following are details of mount; remember "None" represents Python type None - 2024-04-02T04:56:29.593 INFO:tasks.cephfs.mount:self.client_remote.hostname = smithi154.front.sepia.ceph.com 2024-04-02T04:56:29.594 INFO:tasks.cephfs.mount:self.client.name = client.1 2024-04-02T04:56:29.594 INFO:tasks.cephfs.mount:self.hostfs_mntpt = /home/ubuntu/cephtest/mnt.1 2024-04-02T04:56:29.594 INFO:tasks.cephfs.mount:self.cephfs_name = None 2024-04-02T04:56:29.594 INFO:tasks.cephfs.mount:self.cephfs_mntpt = None 2024-04-02T04:56:29.594 INFO:tasks.cephfs.mount:self.client_keyring_path = None 2024-04-02T04:56:29.594 INFO:tasks.cephfs.mount:Setting the 'None' netns for '/home/ubuntu/cephtest/mnt.1' 2024-04-02T04:56:29.594 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-02T04:56:29.594 DEBUG:teuthology.orchestra.run.smithi154:> ip addr 2024-04-02T04:56:29.613 INFO:teuthology.orchestra.run.smithi154.stdout:1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 2024-04-02T04:56:29.613 INFO:teuthology.orchestra.run.smithi154.stdout: link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 2024-04-02T04:56:29.614 INFO:teuthology.orchestra.run.smithi154.stdout: inet 127.0.0.1/8 scope host lo 2024-04-02T04:56:29.614 INFO:teuthology.orchestra.run.smithi154.stdout: valid_lft forever preferred_lft forever 2024-04-02T04:56:29.614 INFO:teuthology.orchestra.run.smithi154.stdout: inet6 ::1/128 scope host 2024-04-02T04:56:29.614 INFO:teuthology.orchestra.run.smithi154.stdout: valid_lft forever preferred_lft forever 2024-04-02T04:56:29.614 INFO:teuthology.orchestra.run.smithi154.stdout:2: eno1: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-04-02T04:56:29.614 INFO:teuthology.orchestra.run.smithi154.stdout: link/ether 0c:c4:7a:d9:1a:54 brd ff:ff:ff:ff:ff:ff 2024-04-02T04:56:29.614 INFO:teuthology.orchestra.run.smithi154.stdout: altname enp7s0f0 2024-04-02T04:56:29.614 INFO:teuthology.orchestra.run.smithi154.stdout:3: eno2: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-04-02T04:56:29.614 INFO:teuthology.orchestra.run.smithi154.stdout: link/ether 0c:c4:7a:d9:1a:55 brd ff:ff:ff:ff:ff:ff 2024-04-02T04:56:29.614 INFO:teuthology.orchestra.run.smithi154.stdout: altname enp7s0f1 2024-04-02T04:56:29.614 INFO:teuthology.orchestra.run.smithi154.stdout:4: enp3s0f0: mtu 1500 qdisc mq state DOWN group default qlen 1000 2024-04-02T04:56:29.614 INFO:teuthology.orchestra.run.smithi154.stdout: link/ether 0c:c4:7a:88:77:60 brd ff:ff:ff:ff:ff:ff 2024-04-02T04:56:29.614 INFO:teuthology.orchestra.run.smithi154.stdout:5: enp3s0f1: mtu 1500 qdisc mq state UP group default qlen 1000 2024-04-02T04:56:29.615 INFO:teuthology.orchestra.run.smithi154.stdout: link/ether 0c:c4:7a:88:77:61 brd ff:ff:ff:ff:ff:ff 2024-04-02T04:56:29.615 INFO:teuthology.orchestra.run.smithi154.stdout: inet 172.21.15.154/20 brd 172.21.15.255 scope global dynamic noprefixroute enp3s0f1 2024-04-02T04:56:29.615 INFO:teuthology.orchestra.run.smithi154.stdout: valid_lft 42265sec preferred_lft 42265sec 2024-04-02T04:56:29.615 INFO:teuthology.orchestra.run.smithi154.stdout: inet6 fe80::ec4:7aff:fe88:7761/64 scope link 2024-04-02T04:56:29.615 INFO:teuthology.orchestra.run.smithi154.stdout: valid_lft forever preferred_lft forever 2024-04-02T04:56:29.615 INFO:tasks.cephfs.mount:Setuping the 'ceph-brx' with 192.168.255.254/16 2024-04-02T04:56:29.615 DEBUG:teuthology.orchestra.run.smithi154:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-04-02T04:56:29.615 DEBUG:teuthology.orchestra.run.smithi154:> set -e 2024-04-02T04:56:29.615 DEBUG:teuthology.orchestra.run.smithi154:> sudo ip link add name ceph-brx type bridge 2024-04-02T04:56:29.615 DEBUG:teuthology.orchestra.run.smithi154:> sudo ip addr flush dev ceph-brx 2024-04-02T04:56:29.616 DEBUG:teuthology.orchestra.run.smithi154:> sudo ip link set ceph-brx up 2024-04-02T04:56:29.616 DEBUG:teuthology.orchestra.run.smithi154:> sudo ip addr add 192.168.255.254/16 brd 192.168.255.255 dev ceph-brx 2024-04-02T04:56:29.616 DEBUG:teuthology.orchestra.run.smithi154:> ') 2024-04-02T04:56:29.709 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:56:29 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-04-02T04:56:29.823 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:56:29 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-02T04:56:29.829 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-02T04:56:29.829 DEBUG:teuthology.orchestra.run.smithi154:> echo 1 | sudo tee /proc/sys/net/ipv4/ip_forward 2024-04-02T04:56:29.857 INFO:teuthology.orchestra.run.smithi154.stdout:1 2024-04-02T04:56:29.859 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-02T04:56:29.859 DEBUG:teuthology.orchestra.run.smithi154:> route 2024-04-02T04:56:29.926 INFO:teuthology.orchestra.run.smithi154.stdout:Kernel IP routing table 2024-04-02T04:56:29.926 INFO:teuthology.orchestra.run.smithi154.stdout:Destination Gateway Genmask Flags Metric Ref Use Iface 2024-04-02T04:56:29.926 INFO:teuthology.orchestra.run.smithi154.stdout:default _gateway 0.0.0.0 UG 100 0 0 enp3s0f1 2024-04-02T04:56:29.926 INFO:teuthology.orchestra.run.smithi154.stdout:172.21.0.0 0.0.0.0 255.255.240.0 U 100 0 0 enp3s0f1 2024-04-02T04:56:29.926 INFO:teuthology.orchestra.run.smithi154.stdout:192.168.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ceph-brx 2024-04-02T04:56:29.927 DEBUG:teuthology.orchestra.run.smithi154:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-04-02T04:56:29.927 DEBUG:teuthology.orchestra.run.smithi154:> set -e 2024-04-02T04:56:29.927 DEBUG:teuthology.orchestra.run.smithi154:> sudo iptables -A FORWARD -o enp3s0f1 -i ceph-brx -j ACCEPT 2024-04-02T04:56:29.927 DEBUG:teuthology.orchestra.run.smithi154:> sudo iptables -A FORWARD -i enp3s0f1 -o ceph-brx -j ACCEPT 2024-04-02T04:56:29.927 DEBUG:teuthology.orchestra.run.smithi154:> sudo iptables -t nat -A POSTROUTING -s 192.168.255.254/16 -o enp3s0f1 -j MASQUERADE 2024-04-02T04:56:29.927 DEBUG:teuthology.orchestra.run.smithi154:> ') 2024-04-02T04:56:30.016 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:56:30 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-04-02T04:56:30.230 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:56:30 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-02T04:56:30.237 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-02T04:56:30.237 DEBUG:teuthology.orchestra.run.smithi154:> ip netns list 2024-04-02T04:56:30.252 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-02T04:56:30.252 DEBUG:teuthology.orchestra.run.smithi154:> ip netns list-id 2024-04-02T04:56:30.307 DEBUG:teuthology.orchestra.run.smithi154:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-04-02T04:56:30.307 DEBUG:teuthology.orchestra.run.smithi154:> set -e 2024-04-02T04:56:30.307 DEBUG:teuthology.orchestra.run.smithi154:> sudo ip netns add ceph-ns--home-ubuntu-cephtest-mnt.1 2024-04-02T04:56:30.307 DEBUG:teuthology.orchestra.run.smithi154:> sudo ip netns set ceph-ns--home-ubuntu-cephtest-mnt.1 0 2024-04-02T04:56:30.308 DEBUG:teuthology.orchestra.run.smithi154:> ') 2024-04-02T04:56:30.406 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:56:30 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-04-02T04:56:30.436 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:56:30 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-02T04:56:30.442 INFO:tasks.cephfs.mount:Setuping the netns 'ceph-ns--home-ubuntu-cephtest-mnt.1' with 192.168.0.1/16 2024-04-02T04:56:30.442 DEBUG:teuthology.orchestra.run.smithi154:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-04-02T04:56:30.442 DEBUG:teuthology.orchestra.run.smithi154:> set -e 2024-04-02T04:56:30.442 DEBUG:teuthology.orchestra.run.smithi154:> sudo ip link add veth0 netns ceph-ns--home-ubuntu-cephtest-mnt.1 type veth peer name brx.0 2024-04-02T04:56:30.442 DEBUG:teuthology.orchestra.run.smithi154:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.1 ip addr add 192.168.0.1/16 brd 192.168.255.255 dev veth0 2024-04-02T04:56:30.443 DEBUG:teuthology.orchestra.run.smithi154:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.1 ip link set veth0 up 2024-04-02T04:56:30.443 DEBUG:teuthology.orchestra.run.smithi154:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.1 ip link set lo up 2024-04-02T04:56:30.443 DEBUG:teuthology.orchestra.run.smithi154:> sudo ip netns exec ceph-ns--home-ubuntu-cephtest-mnt.1 ip route add default via 192.168.255.254 2024-04-02T04:56:30.443 DEBUG:teuthology.orchestra.run.smithi154:> ') 2024-04-02T04:56:30.493 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:56:30 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-04-02T04:56:30.580 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:56:30 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-02T04:56:30.586 DEBUG:teuthology.orchestra.run.smithi154:> (cd / && exec stdin-killer --timeout=300 -- bash -c ' 2024-04-02T04:56:30.586 DEBUG:teuthology.orchestra.run.smithi154:> set -e 2024-04-02T04:56:30.586 DEBUG:teuthology.orchestra.run.smithi154:> sudo ip link set brx.0 up 2024-04-02T04:56:30.586 DEBUG:teuthology.orchestra.run.smithi154:> sudo ip link set dev brx.0 master ceph-brx 2024-04-02T04:56:30.586 DEBUG:teuthology.orchestra.run.smithi154:> ') 2024-04-02T04:56:30.634 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:56:30 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-04-02T04:56:30.662 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:56:30 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-02T04:56:30.668 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-04-02T04:56:30.668 DEBUG:teuthology.orchestra.run.smithi154:> mkdir -p -v /home/ubuntu/cephtest/mnt.1 2024-04-02T04:56:30.683 INFO:teuthology.orchestra.run.smithi154.stdout:mkdir: created directory '/home/ubuntu/cephtest/mnt.1' 2024-04-02T04:56:30.683 INFO:teuthology.orchestra.run:Running command with timeout 60 2024-04-02T04:56:30.683 DEBUG:teuthology.orchestra.run.smithi154:> chmod 0000 /home/ubuntu/cephtest/mnt.1 2024-04-02T04:56:30.738 INFO:tasks.cephfs.kernel_mount:mounting using device: :/ 2024-04-02T04:56:30.738 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-02T04:56:30.738 DEBUG:teuthology.orchestra.run.smithi154:> sudo nsenter --net=/var/run/netns/ceph-ns--home-ubuntu-cephtest-mnt.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/mount -t ceph :/ /home/ubuntu/cephtest/mnt.1 -v -o norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,name=1,mds_namespace=cephfs 2024-04-02T04:56:30.838 INFO:teuthology.orchestra.run.smithi154.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,name=1,mds_namespace=cephfs 2024-04-02T04:56:30.838 INFO:teuthology.orchestra.run.smithi154.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=1,mds_namespace=cephfs". 2024-04-02T04:56:30.839 INFO:teuthology.orchestra.run.smithi154.stdout:invalid new device string format 2024-04-02T04:56:30.951 INFO:teuthology.orchestra.run.smithi154.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,name=1,mds_namespace=cephfs 2024-04-02T04:56:30.952 INFO:teuthology.orchestra.run.smithi154.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=1,mds_namespace=cephfs". 2024-04-02T04:56:30.952 INFO:teuthology.orchestra.run.smithi154.stdout:invalid new device string format 2024-04-02T04:56:30.952 INFO:teuthology.orchestra.run.smithi154.stdout:mount.ceph: resolved to: "172.21.15.152:3300,172.21.15.154:3300" 2024-04-02T04:56:30.952 INFO:teuthology.orchestra.run.smithi154.stdout:mount.ceph: switching to using v1 address with old syntax 2024-04-02T04:56:30.956 INFO:teuthology.orchestra.run.smithi154.stdout:parsing options: rw,norequire_active_mds,conf=/etc/ceph/ceph.conf,norbytes,name=1,mds_namespace=cephfs 2024-04-02T04:56:30.956 INFO:teuthology.orchestra.run.smithi154.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=1,mds_namespace=cephfs". 2024-04-02T04:56:30.956 INFO:teuthology.orchestra.run.smithi154.stdout:invalid new device string format 2024-04-02T04:56:30.956 INFO:teuthology.orchestra.run.smithi154.stdout:mount.ceph: resolved to: "172.21.15.152:3300,172.21.15.154:3300" 2024-04-02T04:56:30.956 INFO:teuthology.orchestra.run.smithi154.stdout:mount.ceph: switching to using v1 address with old syntax 2024-04-02T04:56:30.956 INFO:teuthology.orchestra.run.smithi154.stdout:mount.ceph: resolved to: "172.21.15.152:6789,172.21.15.154:6789" 2024-04-02T04:56:30.956 INFO:teuthology.orchestra.run.smithi154.stdout:mount.ceph: trying mount with old device syntax: 172.21.15.152:6789,172.21.15.154:6789:/ 2024-04-02T04:56:30.956 INFO:teuthology.orchestra.run.smithi154.stdout:mount.ceph: options "norequire_active_mds,norbytes,name=1,mds_namespace=cephfs,key=1,fsid=2e0055d0-f0ac-11ee-b647-cb9ed24678a4" will pass to kernel 2024-04-02T04:56:30.960 INFO:teuthology.orchestra.run.smithi154.stdout:mount: /home/ubuntu/cephtest/mnt.1 does not contain SELinux labels. 2024-04-02T04:56:30.960 INFO:teuthology.orchestra.run.smithi154.stdout: You just mounted a file system that supports labels which does not 2024-04-02T04:56:30.960 INFO:teuthology.orchestra.run.smithi154.stdout: contain labels, onto an SELinux box. It is likely that confined 2024-04-02T04:56:30.961 INFO:teuthology.orchestra.run.smithi154.stdout: applications will generate AVC messages and not be allowed access to 2024-04-02T04:56:30.961 INFO:teuthology.orchestra.run.smithi154.stdout: this file system. For more details see restorecon(8) and mount(8). 2024-04-02T04:56:30.961 INFO:tasks.cephfs.kernel_mount:mount command passed 2024-04-02T04:56:30.962 INFO:teuthology.orchestra.run:Running command with timeout 300 2024-04-02T04:56:30.962 DEBUG:teuthology.orchestra.run.smithi154:> sudo chmod 1777 /home/ubuntu/cephtest/mnt.1 2024-04-02T04:56:30.988 DEBUG:teuthology.orchestra.run.smithi154:> (cd /home/ubuntu/cephtest/mnt.1 && exec stdin-killer --timeout=900 -- bash -c 'getfattr --only-values -n ceph.client_id .') 2024-04-02T04:56:31.078 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:56:31 stdin-killer INFO: expiration expected; waiting 900 seconds for command to complete 2024-04-02T04:56:31.080 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:56:31 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-02T04:56:31.085 INFO:teuthology.orchestra.run.smithi154.stdout:client24337 2024-04-02T04:56:31.086 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-04-02T04:56:31.086 DEBUG:teuthology.orchestra.run.smithi154:> dd if=/proc/self/mounts of=/dev/stdout 2024-04-02T04:56:31.102 DEBUG:teuthology.orchestra.run.smithi154:> (cd /home/ubuntu/cephtest/mnt.1 && exec stdin-killer --timeout=900 -- bash -c 'getfattr --only-values -n ceph.client_id .') 2024-04-02T04:56:31.191 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:56:31 stdin-killer INFO: expiration expected; waiting 900 seconds for command to complete 2024-04-02T04:56:31.192 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:56:31 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-02T04:56:31.198 INFO:teuthology.orchestra.run.smithi154.stdout:client24337 2024-04-02T04:56:31.198 DEBUG:teuthology.orchestra.run.smithi154:> (cd /home/ubuntu/cephtest/mnt.1 && exec stdin-killer --timeout=900 -- bash -c 'getfattr --only-values -n ceph.client_id .') 2024-04-02T04:56:31.247 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:56:31 stdin-killer INFO: expiration expected; waiting 900 seconds for command to complete 2024-04-02T04:56:31.249 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:56:31 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-02T04:56:31.255 INFO:teuthology.orchestra.run.smithi154.stdout:client24337 2024-04-02T04:56:31.255 DEBUG:teuthology.orchestra.run.smithi154:> (cd /home/ubuntu/cephtest/mnt.1 && exec stdin-killer --timeout=300 -- bash -c 'sudo dd if=/sys/kernel/debug/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4.client24337/status') 2024-04-02T04:56:31.303 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:56:31 stdin-killer INFO: expiration expected; waiting 300 seconds for command to complete 2024-04-02T04:56:31.317 INFO:teuthology.orchestra.run.smithi154.stdout:instance: client.24337 (3)192.168.0.1:0/2897042683 2024-04-02T04:56:31.317 INFO:teuthology.orchestra.run.smithi154.stdout:blocklisted: false 2024-04-02T04:56:31.317 INFO:teuthology.orchestra.run.smithi154.stderr:0+1 records in 2024-04-02T04:56:31.317 INFO:teuthology.orchestra.run.smithi154.stderr:0+1 records out 2024-04-02T04:56:31.317 INFO:teuthology.orchestra.run.smithi154.stderr:70 bytes copied, 2.4654e-05 s, 2.8 MB/s 2024-04-02T04:56:31.318 INFO:teuthology.orchestra.run.smithi154.stderr:2024-04-02T04:56:31 stdin-killer INFO: command exited with status 0: exiting normally with same code! 2024-04-02T04:56:31.324 INFO:teuthology.run_tasks:Running task print... 2024-04-02T04:56:31.334 INFO:teuthology.task.print:**** done client 2024-04-02T04:56:31.334 INFO:teuthology.run_tasks:Running task parallel... 2024-04-02T04:56:31.344 INFO:teuthology.task.parallel:starting parallel... 2024-04-02T04:56:31.345 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-02T04:56:31.345 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-02T04:56:31.345 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi152.front.sepia.ceph.com 2024-04-02T04:56:31.346 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph config set mgr mgr/orchestrator/fail_fs false || true' 2024-04-02T04:56:31.346 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-02T04:56:31.346 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-04-02T04:56:31.349 INFO:tasks.workunit:Pulling workunits from ref 6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T04:56:31.350 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-04-02T04:56:31.350 INFO:tasks.workunit:timeout=3h 2024-04-02T04:56:31.350 INFO:tasks.workunit:cleanup=True 2024-04-02T04:56:31.350 DEBUG:teuthology.orchestra.run.smithi152:> stat -- /home/ubuntu/cephtest/mnt.0 2024-04-02T04:56:31.368 INFO:teuthology.orchestra.run.smithi152.stdout: File: /home/ubuntu/cephtest/mnt.0 2024-04-02T04:56:31.368 INFO:teuthology.orchestra.run.smithi152.stdout: Size: 0 Blocks: 0 IO Block: 65536 directory 2024-04-02T04:56:31.368 INFO:teuthology.orchestra.run.smithi152.stdout:Device: 94h/148d Inode: 1 Links: 2 2024-04-02T04:56:31.368 INFO:teuthology.orchestra.run.smithi152.stdout:Access: (1777/drwxrwxrwt) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T04:56:31.368 INFO:teuthology.orchestra.run.smithi152.stdout:Context: system_u:object_r:unlabeled_t:s0 2024-04-02T04:56:31.368 INFO:teuthology.orchestra.run.smithi152.stdout:Access: 1970-01-01 00:00:00.000000000 +0000 2024-04-02T04:56:31.368 INFO:teuthology.orchestra.run.smithi152.stdout:Modify: 2024-04-02 04:55:54.469116782 +0000 2024-04-02T04:56:31.368 INFO:teuthology.orchestra.run.smithi152.stdout:Change: 2024-04-02 04:56:26.445900825 +0000 2024-04-02T04:56:31.368 INFO:teuthology.orchestra.run.smithi152.stdout: Birth: 2024-04-02 04:55:54.469116782 +0000 2024-04-02T04:56:31.368 INFO:tasks.workunit:Did not need to create dir /home/ubuntu/cephtest/mnt.0 2024-04-02T04:56:31.368 DEBUG:teuthology.orchestra.run.smithi152:> cd -- /home/ubuntu/cephtest/mnt.0 && sudo install -d -m 0755 --owner=ubuntu -- client.0 2024-04-02T04:56:31.377 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:31 smithi152 ceph-mon[29073]: pgmap v194: 65 pgs: 65 active+clean; 581 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.7 KiB/s rd, 341 B/s wr, 5 op/s 2024-04-02T04:56:31.409 DEBUG:teuthology.orchestra.run.smithi154:> stat -- /home/ubuntu/cephtest/mnt.1 2024-04-02T04:56:31.425 INFO:teuthology.orchestra.run.smithi154.stdout: File: /home/ubuntu/cephtest/mnt.1 2024-04-02T04:56:31.425 INFO:teuthology.orchestra.run.smithi154.stdout: Size: 1 Blocks: 0 IO Block: 65536 directory 2024-04-02T04:56:31.425 INFO:teuthology.orchestra.run.smithi154.stdout:Device: 73h/115d Inode: 1 Links: 3 2024-04-02T04:56:31.425 INFO:teuthology.orchestra.run.smithi154.stdout:Access: (1777/drwxrwxrwt) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-02T04:56:31.425 INFO:teuthology.orchestra.run.smithi154.stdout:Context: system_u:object_r:unlabeled_t:s0 2024-04-02T04:56:31.425 INFO:teuthology.orchestra.run.smithi154.stdout:Access: 1970-01-01 00:00:00.000000000 +0000 2024-04-02T04:56:31.425 INFO:teuthology.orchestra.run.smithi154.stdout:Modify: 2024-04-02 04:56:31.400838934 +0000 2024-04-02T04:56:31.425 INFO:teuthology.orchestra.run.smithi154.stdout:Change: 2024-04-02 04:56:31.400838934 +0000 2024-04-02T04:56:31.426 INFO:teuthology.orchestra.run.smithi154.stdout: Birth: 2024-04-02 04:55:54.469116782 +0000 2024-04-02T04:56:31.426 INFO:tasks.workunit:Did not need to create dir /home/ubuntu/cephtest/mnt.1 2024-04-02T04:56:31.426 DEBUG:teuthology.orchestra.run.smithi154:> cd -- /home/ubuntu/cephtest/mnt.1 && sudo install -d -m 0755 --owner=ubuntu -- client.1 2024-04-02T04:56:31.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:31 smithi154 ceph-mon[35729]: pgmap v194: 65 pgs: 65 active+clean; 581 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.7 KiB/s rd, 341 B/s wr, 5 op/s 2024-04-02T04:56:31.496 DEBUG:teuthology.orchestra.run.smithi152:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 && git checkout 6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T04:56:31.497 DEBUG:teuthology.orchestra.run.smithi154:> rm -rf /home/ubuntu/cephtest/clone.client.1 && git clone https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.1 && cd /home/ubuntu/cephtest/clone.client.1 && git checkout 6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T04:56:31.512 INFO:tasks.workunit.client.0.smithi152.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-04-02T04:56:31.552 INFO:tasks.workunit.client.1.smithi154.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.1'... 2024-04-02T04:56:31.617 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:56:33.211 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.204+0000 7f486baed700 1 -- 172.21.15.152:0/2835677949 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f48640fed90 msgr2=0x7f48640faed0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:33.211 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.204+0000 7f486baed700 1 --2- 172.21.15.152:0/2835677949 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f48640fed90 0x7f48640faed0 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 crypto rx=0x7f4854004940 tx=0x7f4854009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:33.211 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.204+0000 7f486baed700 1 -- 172.21.15.152:0/2835677949 shutdown_connections 2024-04-02T04:56:33.211 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.204+0000 7f486baed700 1 --2- 172.21.15.152:0/2835677949 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f48640fed90 0x7f48640faed0 unknown :-1 s=CLOSED pgs=266 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:33.212 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.204+0000 7f486baed700 1 --2- 172.21.15.152:0/2835677949 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f48640fe440 0x7f48640fe850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:33.212 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.204+0000 7f486baed700 1 -- 172.21.15.152:0/2835677949 >> 172.21.15.152:0/2835677949 conn(0x7f48640f6be0 msgr2=0x7f48640f9010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:33.212 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.204+0000 7f486baed700 1 -- 172.21.15.152:0/2835677949 shutdown_connections 2024-04-02T04:56:33.212 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.204+0000 7f486baed700 1 -- 172.21.15.152:0/2835677949 wait complete. 2024-04-02T04:56:33.212 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.205+0000 7f486baed700 1 Processor -- start 2024-04-02T04:56:33.212 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.205+0000 7f486baed700 1 -- start start 2024-04-02T04:56:33.212 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.205+0000 7f486baed700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f48640fe440 0x7f48640fda50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:33.212 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.205+0000 7f486baed700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f48640fed90 0x7f48640fc100 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:33.212 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.205+0000 7f486baed700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f48640fe050 con 0x7f48640fe440 2024-04-02T04:56:33.212 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.205+0000 7f486baed700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f48640fc640 con 0x7f48640fed90 2024-04-02T04:56:33.212 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.205+0000 7f4869889700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f48640fe440 0x7f48640fda50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:33.212 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.205+0000 7f4869889700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f48640fe440 0x7f48640fda50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:39410/0 (socket says 172.21.15.152:39410) 2024-04-02T04:56:33.213 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.205+0000 7f4869889700 1 -- 172.21.15.152:0/490124779 learned_addr learned my addr 172.21.15.152:0/490124779 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:56:33.213 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.205+0000 7f4869088700 1 --2- 172.21.15.152:0/490124779 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f48640fed90 0x7f48640fc100 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:33.213 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.206+0000 7f4869889700 1 -- 172.21.15.152:0/490124779 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f48640fed90 msgr2=0x7f48640fc100 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:33.213 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.206+0000 7f4869889700 1 --2- 172.21.15.152:0/490124779 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f48640fed90 0x7f48640fc100 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:33.213 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.206+0000 7f4869889700 1 -- 172.21.15.152:0/490124779 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4854005040 con 0x7f48640fe440 2024-04-02T04:56:33.213 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.206+0000 7f4869889700 1 --2- 172.21.15.152:0/490124779 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f48640fe440 0x7f48640fda50 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 crypto rx=0x7f486000c8c0 tx=0x7f48600044c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:33.213 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.206+0000 7f485a7fc700 1 -- 172.21.15.152:0/490124779 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f486000a760 con 0x7f48640fe440 2024-04-02T04:56:33.213 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.206+0000 7f486baed700 1 -- 172.21.15.152:0/490124779 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f48640fc8f0 con 0x7f48640fe440 2024-04-02T04:56:33.213 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.206+0000 7f485a7fc700 1 -- 172.21.15.152:0/490124779 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f486000ad90 con 0x7f48640fe440 2024-04-02T04:56:33.213 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.206+0000 7f485a7fc700 1 -- 172.21.15.152:0/490124779 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f48600113a0 con 0x7f48640fe440 2024-04-02T04:56:33.213 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.206+0000 7f486baed700 1 -- 172.21.15.152:0/490124779 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f48640fceb0 con 0x7f48640fe440 2024-04-02T04:56:33.213 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.207+0000 7f486baed700 1 -- 172.21.15.152:0/490124779 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f486404c710 con 0x7f48640fe440 2024-04-02T04:56:33.213 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.207+0000 7f485a7fc700 1 -- 172.21.15.152:0/490124779 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f4860012400 con 0x7f48640fe440 2024-04-02T04:56:33.214 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.208+0000 7f485a7fc700 1 --2- 172.21.15.152:0/490124779 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f48500692a0 0x7f485006b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:33.214 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.208+0000 7f485a7fc700 1 -- 172.21.15.152:0/490124779 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7f48600871f0 con 0x7f48640fe440 2024-04-02T04:56:33.214 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.208+0000 7f4869088700 1 --2- 172.21.15.152:0/490124779 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f48500692a0 0x7f485006b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:33.214 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.208+0000 7f4869088700 1 --2- 172.21.15.152:0/490124779 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f48500692a0 0x7f485006b750 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 crypto rx=0x7f4854009fd0 tx=0x7f4854009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:33.216 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.212+0000 7f485a7fc700 1 -- 172.21.15.152:0/490124779 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f4860056f60 con 0x7f48640fe440 2024-04-02T04:56:33.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:33 smithi152 ceph-mon[29073]: pgmap v195: 65 pgs: 65 active+clean; 581 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.8 KiB/s rd, 341 B/s wr, 4 op/s 2024-04-02T04:56:33.407 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.403+0000 7f486baed700 1 -- 172.21.15.152:0/490124779 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command([{prefix=config set, name=mgr/orchestrator/fail_fs}] v 0) v1 -- 0x7f48641058f0 con 0x7f48640fe440 2024-04-02T04:56:33.408 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:33.404+0000 7f485a7fc700 1 -- 172.21.15.152:0/490124779 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/orchestrator/fail_fs}]=-22 unrecognized config option 'mgr/orchestrator/fail_fs' v21)=-22 unrecognized config option 'mgr/orchestrator/fail_fs' v21) v1 ==== 179+0+0 (secure 0 0 0) 0x7f4860056af0 con 0x7f48640fe440 2024-04-02T04:56:33.409 INFO:teuthology.orchestra.run.smithi152.stderr:Error EINVAL: unrecognized config option 'mgr/orchestrator/fail_fs' 2024-04-02T04:56:33.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:33 smithi154 ceph-mon[35729]: pgmap v195: 65 pgs: 65 active+clean; 581 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.8 KiB/s rd, 341 B/s wr, 4 op/s 2024-04-02T04:56:34.091 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-02T04:56:34.092 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi152.front.sepia.ceph.com 2024-04-02T04:56:34.092 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-02T04:56:34.390 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:56:35.182 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:35 smithi152 ceph-mon[29073]: pgmap v196: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 3.2 KiB/s rd, 1023 B/s wr, 5 op/s 2024-04-02T04:56:35.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:35 smithi154 ceph-mon[35729]: pgmap v196: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 3.2 KiB/s rd, 1023 B/s wr, 5 op/s 2024-04-02T04:56:36.034 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.028+0000 7f0f27181700 1 -- 172.21.15.152:0/1876951865 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0f200fbcd0 msgr2=0x7f0f200fc0e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:36.034 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.028+0000 7f0f27181700 1 --2- 172.21.15.152:0/1876951865 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0f200fbcd0 0x7f0f200fc0e0 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 crypto rx=0x7f0f14004940 tx=0x7f0f14009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:36.034 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.028+0000 7f0f27181700 1 -- 172.21.15.152:0/1876951865 shutdown_connections 2024-04-02T04:56:36.034 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.028+0000 7f0f27181700 1 --2- 172.21.15.152:0/1876951865 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0f200fcf80 0x7f0f200fd3f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:36.034 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.028+0000 7f0f27181700 1 --2- 172.21.15.152:0/1876951865 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0f200fbcd0 0x7f0f200fc0e0 unknown :-1 s=CLOSED pgs=268 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:36.035 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.028+0000 7f0f27181700 1 -- 172.21.15.152:0/1876951865 >> 172.21.15.152:0/1876951865 conn(0x7f0f200f7450 msgr2=0x7f0f200f9860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:36.035 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.028+0000 7f0f27181700 1 -- 172.21.15.152:0/1876951865 shutdown_connections 2024-04-02T04:56:36.035 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.028+0000 7f0f27181700 1 -- 172.21.15.152:0/1876951865 wait complete. 2024-04-02T04:56:36.035 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.029+0000 7f0f27181700 1 Processor -- start 2024-04-02T04:56:36.035 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.029+0000 7f0f27181700 1 -- start start 2024-04-02T04:56:36.035 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.029+0000 7f0f27181700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0f200fbcd0 0x7f0f200ffb70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:36.035 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.029+0000 7f0f27181700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0f200fcf80 0x7f0f200fe2c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:36.035 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.029+0000 7f0f27181700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0f20100200 con 0x7f0f200fcf80 2024-04-02T04:56:36.035 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.029+0000 7f0f27181700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0f200fe800 con 0x7f0f200fbcd0 2024-04-02T04:56:36.035 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.029+0000 7f0f1ffff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0f200fcf80 0x7f0f200fe2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:36.035 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.029+0000 7f0f1ffff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0f200fcf80 0x7f0f200fe2c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:39428/0 (socket says 172.21.15.152:39428) 2024-04-02T04:56:36.035 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.029+0000 7f0f1ffff700 1 -- 172.21.15.152:0/304599700 learned_addr learned my addr 172.21.15.152:0/304599700 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:56:36.035 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.030+0000 7f0f24f1d700 1 --2- 172.21.15.152:0/304599700 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0f200fbcd0 0x7f0f200ffb70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:36.036 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.030+0000 7f0f1ffff700 1 -- 172.21.15.152:0/304599700 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0f200fbcd0 msgr2=0x7f0f200ffb70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:36.036 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.030+0000 7f0f1ffff700 1 --2- 172.21.15.152:0/304599700 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0f200fbcd0 0x7f0f200ffb70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:36.036 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.030+0000 7f0f1ffff700 1 -- 172.21.15.152:0/304599700 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0f14005040 con 0x7f0f200fcf80 2024-04-02T04:56:36.036 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.030+0000 7f0f1ffff700 1 --2- 172.21.15.152:0/304599700 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0f200fcf80 0x7f0f200fe2c0 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7f0f0c00b790 tx=0x7f0f0c009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:36.036 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.030+0000 7f0f1d7fa700 1 -- 172.21.15.152:0/304599700 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f0f0c00f6f0 con 0x7f0f200fcf80 2024-04-02T04:56:36.036 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.030+0000 7f0f27181700 1 -- 172.21.15.152:0/304599700 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0f200feae0 con 0x7f0f200fcf80 2024-04-02T04:56:36.036 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.030+0000 7f0f1d7fa700 1 -- 172.21.15.152:0/304599700 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f0f0c00fd20 con 0x7f0f200fcf80 2024-04-02T04:56:36.036 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.030+0000 7f0f1d7fa700 1 -- 172.21.15.152:0/304599700 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f0f0c0103a0 con 0x7f0f200fcf80 2024-04-02T04:56:36.036 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.030+0000 7f0f27181700 1 -- 172.21.15.152:0/304599700 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0f200ff0a0 con 0x7f0f200fcf80 2024-04-02T04:56:36.037 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.031+0000 7f0f27181700 1 -- 172.21.15.152:0/304599700 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0f2004c710 con 0x7f0f200fcf80 2024-04-02T04:56:36.037 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.032+0000 7f0f1d7fa700 1 -- 172.21.15.152:0/304599700 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f0f0c011400 con 0x7f0f200fcf80 2024-04-02T04:56:36.037 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.032+0000 7f0f1d7fa700 1 --2- 172.21.15.152:0/304599700 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f0f100692a0 0x7f0f1006b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:36.037 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.032+0000 7f0f1d7fa700 1 -- 172.21.15.152:0/304599700 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7f0f0c086fb0 con 0x7f0f200fcf80 2024-04-02T04:56:36.037 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.032+0000 7f0f24f1d700 1 --2- 172.21.15.152:0/304599700 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f0f100692a0 0x7f0f1006b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:36.037 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.033+0000 7f0f24f1d700 1 --2- 172.21.15.152:0/304599700 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f0f100692a0 0x7f0f1006b750 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f0f1400b3b0 tx=0x7f0f14009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:36.040 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.037+0000 7f0f1d7fa700 1 -- 172.21.15.152:0/304599700 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f0f0c056d20 con 0x7f0f200fcf80 2024-04-02T04:56:36.228 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.224+0000 7f0f27181700 1 -- 172.21.15.152:0/304599700 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command([{prefix=config set, name=mon_warn_on_insecure_global_id_reclaim}] v 0) v1 -- 0x7f0f20101ad0 con 0x7f0f200fcf80 2024-04-02T04:56:36.229 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.225+0000 7f0f1d7fa700 1 -- 172.21.15.152:0/304599700 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mon_warn_on_insecure_global_id_reclaim}]=0 v21)=0 v21) v1 ==== 155+0+0 (secure 0 0 0) 0x7f0f0c0568b0 con 0x7f0f200fcf80 2024-04-02T04:56:36.231 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.227+0000 7f0f27181700 1 -- 172.21.15.152:0/304599700 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f0f100692a0 msgr2=0x7f0f1006b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:36.231 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.227+0000 7f0f27181700 1 --2- 172.21.15.152:0/304599700 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f0f100692a0 0x7f0f1006b750 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 crypto rx=0x7f0f1400b3b0 tx=0x7f0f14009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:36.231 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.227+0000 7f0f27181700 1 -- 172.21.15.152:0/304599700 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0f200fcf80 msgr2=0x7f0f200fe2c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:36.231 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.227+0000 7f0f27181700 1 --2- 172.21.15.152:0/304599700 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0f200fcf80 0x7f0f200fe2c0 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 crypto rx=0x7f0f0c00b790 tx=0x7f0f0c009250 comp rx=0 tx=0).stop 2024-04-02T04:56:36.231 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.227+0000 7f0f27181700 1 -- 172.21.15.152:0/304599700 shutdown_connections 2024-04-02T04:56:36.231 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.227+0000 7f0f27181700 1 --2- 172.21.15.152:0/304599700 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f0f100692a0 0x7f0f1006b750 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:36.231 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.227+0000 7f0f27181700 1 --2- 172.21.15.152:0/304599700 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f0f200fbcd0 0x7f0f200ffb70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:36.231 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.227+0000 7f0f27181700 1 --2- 172.21.15.152:0/304599700 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f0f200fcf80 0x7f0f200fe2c0 unknown :-1 s=CLOSED pgs=269 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:36.231 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.227+0000 7f0f27181700 1 -- 172.21.15.152:0/304599700 >> 172.21.15.152:0/304599700 conn(0x7f0f200f7450 msgr2=0x7f0f20066800 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:36.231 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.227+0000 7f0f27181700 1 -- 172.21.15.152:0/304599700 shutdown_connections 2024-04-02T04:56:36.231 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:36.227+0000 7f0f27181700 1 -- 172.21.15.152:0/304599700 wait complete. 2024-04-02T04:56:36.859 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-02T04:56:37.116 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:56:37.239 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:37 smithi152 ceph-mon[29073]: pgmap v197: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.3 KiB/s rd, 1023 B/s wr, 4 op/s 2024-04-02T04:56:37.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:37 smithi154 ceph-mon[35729]: pgmap v197: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.3 KiB/s rd, 1023 B/s wr, 4 op/s 2024-04-02T04:56:38.787 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.778+0000 7f6362d18700 1 -- 172.21.15.152:0/62634180 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f635c0fcf80 msgr2=0x7f635c0fd3f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:38.787 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.778+0000 7f6362d18700 1 --2- 172.21.15.152:0/62634180 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f635c0fcf80 0x7f635c0fd3f0 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 crypto rx=0x7f6348004940 tx=0x7f6348009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:38.787 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.778+0000 7f6362d18700 1 -- 172.21.15.152:0/62634180 shutdown_connections 2024-04-02T04:56:38.787 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.778+0000 7f6362d18700 1 --2- 172.21.15.152:0/62634180 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f635c0fcf80 0x7f635c0fd3f0 unknown :-1 s=CLOSED pgs=270 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:38.787 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.778+0000 7f6362d18700 1 --2- 172.21.15.152:0/62634180 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f635c0fbcd0 0x7f635c0fc0e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:38.787 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.778+0000 7f6362d18700 1 -- 172.21.15.152:0/62634180 >> 172.21.15.152:0/62634180 conn(0x7f635c0f7430 msgr2=0x7f635c0f9860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:38.787 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.778+0000 7f6362d18700 1 -- 172.21.15.152:0/62634180 shutdown_connections 2024-04-02T04:56:38.787 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.778+0000 7f6362d18700 1 -- 172.21.15.152:0/62634180 wait complete. 2024-04-02T04:56:38.787 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.778+0000 7f6362d18700 1 Processor -- start 2024-04-02T04:56:38.787 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.779+0000 7f6362d18700 1 -- start start 2024-04-02T04:56:38.787 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.779+0000 7f6362d18700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f635c0fbcd0 0x7f635c110710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:38.787 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.779+0000 7f6362d18700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f635c0fcf80 0x7f635c112c60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:38.788 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.779+0000 7f6362d18700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f635c066400 con 0x7f635c0fcf80 2024-04-02T04:56:38.788 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.779+0000 7f6362d18700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f635c066540 con 0x7f635c0fbcd0 2024-04-02T04:56:38.788 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.779+0000 7f635bfff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f635c0fcf80 0x7f635c112c60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:38.788 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.779+0000 7f635bfff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f635c0fcf80 0x7f635c112c60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:39438/0 (socket says 172.21.15.152:39438) 2024-04-02T04:56:38.788 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.779+0000 7f635bfff700 1 -- 172.21.15.152:0/2632148748 learned_addr learned my addr 172.21.15.152:0/2632148748 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:56:38.788 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.779+0000 7f6360ab4700 1 --2- 172.21.15.152:0/2632148748 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f635c0fbcd0 0x7f635c110710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:38.788 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.779+0000 7f635bfff700 1 -- 172.21.15.152:0/2632148748 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f635c0fbcd0 msgr2=0x7f635c110710 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:38.788 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.779+0000 7f635bfff700 1 --2- 172.21.15.152:0/2632148748 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f635c0fbcd0 0x7f635c110710 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:38.788 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.779+0000 7f635bfff700 1 -- 172.21.15.152:0/2632148748 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6348005040 con 0x7f635c0fcf80 2024-04-02T04:56:38.788 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.779+0000 7f635bfff700 1 --2- 172.21.15.152:0/2632148748 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f635c0fcf80 0x7f635c112c60 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7f6348009180 tx=0x7f6348009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:38.788 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.780+0000 7f63597fa700 1 -- 172.21.15.152:0/2632148748 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f634800e040 con 0x7f635c0fcf80 2024-04-02T04:56:38.788 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.780+0000 7f63597fa700 1 -- 172.21.15.152:0/2632148748 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f634800bbf0 con 0x7f635c0fcf80 2024-04-02T04:56:38.788 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.780+0000 7f6362d18700 1 -- 172.21.15.152:0/2632148748 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f635c113260 con 0x7f635c0fcf80 2024-04-02T04:56:38.788 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.780+0000 7f63597fa700 1 -- 172.21.15.152:0/2632148748 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f634801c630 con 0x7f635c0fcf80 2024-04-02T04:56:38.788 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.780+0000 7f6362d18700 1 -- 172.21.15.152:0/2632148748 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f635c113820 con 0x7f635c0fcf80 2024-04-02T04:56:38.789 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.781+0000 7f6362d18700 1 -- 172.21.15.152:0/2632148748 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f635c05f5f0 con 0x7f635c0fcf80 2024-04-02T04:56:38.789 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.781+0000 7f63597fa700 1 -- 172.21.15.152:0/2632148748 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f634800bd60 con 0x7f635c0fcf80 2024-04-02T04:56:38.789 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.781+0000 7f63597fa700 1 --2- 172.21.15.152:0/2632148748 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f634c0692b0 0x7f634c06b760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:38.789 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.782+0000 7f6360ab4700 1 --2- 172.21.15.152:0/2632148748 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f634c0692b0 0x7f634c06b760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:38.789 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.782+0000 7f63597fa700 1 -- 172.21.15.152:0/2632148748 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7f63480884d0 con 0x7f635c0fcf80 2024-04-02T04:56:38.789 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.782+0000 7f6360ab4700 1 --2- 172.21.15.152:0/2632148748 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f634c0692b0 0x7f634c06b760 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f635c065ff0 tx=0x7f63500091a0 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:38.789 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.786+0000 7f63597fa700 1 -- 172.21.15.152:0/2632148748 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f6348054860 con 0x7f635c0fcf80 2024-04-02T04:56:38.973 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.969+0000 7f6362d18700 1 -- 172.21.15.152:0/2632148748 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command([{prefix=config set, name=mon_warn_on_insecure_global_id_reclaim_allowed}] v 0) v1 -- 0x7f635c113a40 con 0x7f635c0fcf80 2024-04-02T04:56:38.974 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.970+0000 7f63597fa700 1 -- 172.21.15.152:0/2632148748 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mon_warn_on_insecure_global_id_reclaim_allowed}]=0 v21)=0 v21) v1 ==== 163+0+0 (secure 0 0 0) 0x7f6348057e80 con 0x7f635c0fcf80 2024-04-02T04:56:38.977 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.973+0000 7f6362d18700 1 -- 172.21.15.152:0/2632148748 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f634c0692b0 msgr2=0x7f634c06b760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:38.977 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.973+0000 7f6362d18700 1 --2- 172.21.15.152:0/2632148748 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f634c0692b0 0x7f634c06b760 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 crypto rx=0x7f635c065ff0 tx=0x7f63500091a0 comp rx=0 tx=0).stop 2024-04-02T04:56:38.977 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.973+0000 7f6362d18700 1 -- 172.21.15.152:0/2632148748 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f635c0fcf80 msgr2=0x7f635c112c60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:38.977 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.973+0000 7f6362d18700 1 --2- 172.21.15.152:0/2632148748 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f635c0fcf80 0x7f635c112c60 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 crypto rx=0x7f6348009180 tx=0x7f6348009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:38.977 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.973+0000 7f6362d18700 1 -- 172.21.15.152:0/2632148748 shutdown_connections 2024-04-02T04:56:38.977 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.973+0000 7f6362d18700 1 --2- 172.21.15.152:0/2632148748 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f634c0692b0 0x7f634c06b760 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:38.977 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.973+0000 7f6362d18700 1 --2- 172.21.15.152:0/2632148748 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f635c0fbcd0 0x7f635c110710 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:38.977 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.973+0000 7f6362d18700 1 --2- 172.21.15.152:0/2632148748 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f635c0fcf80 0x7f635c112c60 unknown :-1 s=CLOSED pgs=271 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:38.977 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.973+0000 7f6362d18700 1 -- 172.21.15.152:0/2632148748 >> 172.21.15.152:0/2632148748 conn(0x7f635c0f7430 msgr2=0x7f635c0f8ab0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:38.978 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.973+0000 7f6362d18700 1 -- 172.21.15.152:0/2632148748 shutdown_connections 2024-04-02T04:56:38.978 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:38.973+0000 7f6362d18700 1 -- 172.21.15.152:0/2632148748 wait complete. 2024-04-02T04:56:39.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:39 smithi152 ceph-mon[29073]: pgmap v198: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.3 KiB/s rd, 1.2 KiB/s wr, 5 op/s 2024-04-02T04:56:39.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:39 smithi154 ceph-mon[35729]: pgmap v198: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.3 KiB/s rd, 1.2 KiB/s wr, 5 op/s 2024-04-02T04:56:39.817 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-02T04:56:40.081 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:56:40.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:40 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:56:40.411 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:40 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:56:41.409 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:41 smithi152 ceph-mon[29073]: pgmap v199: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 3.1 KiB/s rd, 1.1 KiB/s wr, 5 op/s 2024-04-02T04:56:41.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:41 smithi154 ceph-mon[35729]: pgmap v199: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 3.1 KiB/s rd, 1.1 KiB/s wr, 5 op/s 2024-04-02T04:56:41.590 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.585+0000 7f526eab6700 1 -- 172.21.15.152:0/1352686115 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f52680fdc10 msgr2=0x7f52680fc020 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:41.590 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.585+0000 7f526eab6700 1 --2- 172.21.15.152:0/1352686115 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f52680fdc10 0x7f52680fc020 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 crypto rx=0x7f5254004940 tx=0x7f5254009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:41.590 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.585+0000 7f526eab6700 1 -- 172.21.15.152:0/1352686115 shutdown_connections 2024-04-02T04:56:41.590 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.585+0000 7f526eab6700 1 --2- 172.21.15.152:0/1352686115 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f52680fdc10 0x7f52680fc020 unknown :-1 s=CLOSED pgs=272 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:41.590 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.585+0000 7f526eab6700 1 --2- 172.21.15.152:0/1352686115 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f52680fd2c0 0x7f52680fd6d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:41.590 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.585+0000 7f526eab6700 1 -- 172.21.15.152:0/1352686115 >> 172.21.15.152:0/1352686115 conn(0x7f52680f6ba0 msgr2=0x7f52680f8fd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:41.590 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.585+0000 7f526eab6700 1 -- 172.21.15.152:0/1352686115 shutdown_connections 2024-04-02T04:56:41.590 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.586+0000 7f526eab6700 1 -- 172.21.15.152:0/1352686115 wait complete. 2024-04-02T04:56:41.590 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.586+0000 7f526eab6700 1 Processor -- start 2024-04-02T04:56:41.590 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.586+0000 7f526eab6700 1 -- start start 2024-04-02T04:56:41.592 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.586+0000 7f526eab6700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f52680fd2c0 0x7f52681107b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:41.592 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.586+0000 7f526eab6700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f52680fdc10 0x7f5268112d00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:41.592 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.586+0000 7f526eab6700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f52680fbc60 con 0x7f52680fd2c0 2024-04-02T04:56:41.592 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.586+0000 7f526eab6700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f52680fbda0 con 0x7f52680fdc10 2024-04-02T04:56:41.592 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.586+0000 7f526c852700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f52680fd2c0 0x7f52681107b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:41.592 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.586+0000 7f526c852700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f52680fd2c0 0x7f52681107b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:45140/0 (socket says 172.21.15.152:45140) 2024-04-02T04:56:41.592 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.586+0000 7f526c852700 1 -- 172.21.15.152:0/4193766856 learned_addr learned my addr 172.21.15.152:0/4193766856 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:56:41.593 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.587+0000 7f5267fff700 1 --2- 172.21.15.152:0/4193766856 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f52680fdc10 0x7f5268112d00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:41.593 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.587+0000 7f526c852700 1 -- 172.21.15.152:0/4193766856 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f52680fdc10 msgr2=0x7f5268112d00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:41.593 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.587+0000 7f526c852700 1 --2- 172.21.15.152:0/4193766856 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f52680fdc10 0x7f5268112d00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:41.593 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.587+0000 7f526c852700 1 -- 172.21.15.152:0/4193766856 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5254005040 con 0x7f52680fd2c0 2024-04-02T04:56:41.593 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.587+0000 7f526c852700 1 --2- 172.21.15.152:0/4193766856 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f52680fd2c0 0x7f52681107b0 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7f525c00c8c0 tx=0x7f525c0044c0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:41.593 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.587+0000 7f52657fa700 1 -- 172.21.15.152:0/4193766856 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f525c00a760 con 0x7f52680fd2c0 2024-04-02T04:56:41.593 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.587+0000 7f52657fa700 1 -- 172.21.15.152:0/4193766856 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f525c00ad90 con 0x7f52680fd2c0 2024-04-02T04:56:41.593 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.587+0000 7f526eab6700 1 -- 172.21.15.152:0/4193766856 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f52681132a0 con 0x7f52680fd2c0 2024-04-02T04:56:41.593 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.587+0000 7f52657fa700 1 -- 172.21.15.152:0/4193766856 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f525c0113a0 con 0x7f52680fd2c0 2024-04-02T04:56:41.593 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.587+0000 7f526eab6700 1 -- 172.21.15.152:0/4193766856 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5268113860 con 0x7f52680fd2c0 2024-04-02T04:56:41.593 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.588+0000 7f526eab6700 1 -- 172.21.15.152:0/4193766856 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f526805f5f0 con 0x7f52680fd2c0 2024-04-02T04:56:41.593 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.588+0000 7f52657fa700 1 -- 172.21.15.152:0/4193766856 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f525c012400 con 0x7f52680fd2c0 2024-04-02T04:56:41.594 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.589+0000 7f52657fa700 1 --2- 172.21.15.152:0/4193766856 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f52580692a0 0x7f525806b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:41.594 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.589+0000 7f52657fa700 1 -- 172.21.15.152:0/4193766856 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7f525c0871f0 con 0x7f52680fd2c0 2024-04-02T04:56:41.594 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.589+0000 7f5267fff700 1 --2- 172.21.15.152:0/4193766856 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f52580692a0 0x7f525806b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:41.594 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.589+0000 7f5267fff700 1 --2- 172.21.15.152:0/4193766856 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f52580692a0 0x7f525806b750 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f525400b970 tx=0x7f5254009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:41.597 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.593+0000 7f52657fa700 1 -- 172.21.15.152:0/4193766856 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f525c0534d0 con 0x7f52680fd2c0 2024-04-02T04:56:41.786 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.782+0000 7f526eab6700 1 -- 172.21.15.152:0/4193766856 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command([{prefix=config set, name=log_to_journald}] v 0) v1 -- 0x7f526804bed0 con 0x7f52680fd2c0 2024-04-02T04:56:41.786 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.782+0000 7f52657fa700 1 -- 172.21.15.152:0/4193766856 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{prefix=config set, name=log_to_journald}]=0 v21)=0 v21) v1 ==== 135+0+0 (secure 0 0 0) 0x7f525c056af0 con 0x7f52680fd2c0 2024-04-02T04:56:41.788 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.784+0000 7f526eab6700 1 -- 172.21.15.152:0/4193766856 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f52580692a0 msgr2=0x7f525806b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:41.789 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.784+0000 7f526eab6700 1 --2- 172.21.15.152:0/4193766856 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f52580692a0 0x7f525806b750 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 crypto rx=0x7f525400b970 tx=0x7f5254009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:41.789 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.784+0000 7f526eab6700 1 -- 172.21.15.152:0/4193766856 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f52680fd2c0 msgr2=0x7f52681107b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:41.789 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.784+0000 7f526eab6700 1 --2- 172.21.15.152:0/4193766856 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f52680fd2c0 0x7f52681107b0 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 crypto rx=0x7f525c00c8c0 tx=0x7f525c0044c0 comp rx=0 tx=0).stop 2024-04-02T04:56:41.789 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.785+0000 7f526eab6700 1 -- 172.21.15.152:0/4193766856 shutdown_connections 2024-04-02T04:56:41.789 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.785+0000 7f526eab6700 1 --2- 172.21.15.152:0/4193766856 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f52580692a0 0x7f525806b750 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:41.789 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.785+0000 7f526eab6700 1 --2- 172.21.15.152:0/4193766856 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f52680fd2c0 0x7f52681107b0 unknown :-1 s=CLOSED pgs=273 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:41.789 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.785+0000 7f526eab6700 1 --2- 172.21.15.152:0/4193766856 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f52680fdc10 0x7f5268112d00 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:41.789 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.785+0000 7f526eab6700 1 -- 172.21.15.152:0/4193766856 >> 172.21.15.152:0/4193766856 conn(0x7f52680f6ba0 msgr2=0x7f52680f8320 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:41.789 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.785+0000 7f526eab6700 1 -- 172.21.15.152:0/4193766856 shutdown_connections 2024-04-02T04:56:41.789 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:41.785+0000 7f526eab6700 1 -- 172.21.15.152:0/4193766856 wait complete. 2024-04-02T04:56:42.606 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-02T04:56:42.871 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:56:43.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:43 smithi152 ceph-mon[29073]: pgmap v200: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.2 KiB/s rd, 853 B/s wr, 4 op/s 2024-04-02T04:56:43.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:43 smithi154 ceph-mon[35729]: pgmap v200: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.2 KiB/s rd, 853 B/s wr, 4 op/s 2024-04-02T04:56:44.683 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.678+0000 7fa555b99700 1 -- 172.21.15.152:0/3519459672 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fa5500fbd00 msgr2=0x7fa5500fc110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:44.683 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.678+0000 7fa555b99700 1 --2- 172.21.15.152:0/3519459672 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fa5500fbd00 0x7fa5500fc110 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 crypto rx=0x7fa540004940 tx=0x7fa540009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:44.683 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.678+0000 7fa555b99700 1 -- 172.21.15.152:0/3519459672 shutdown_connections 2024-04-02T04:56:44.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.678+0000 7fa555b99700 1 --2- 172.21.15.152:0/3519459672 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa5500fcfb0 0x7fa5500fd420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:44.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.678+0000 7fa555b99700 1 --2- 172.21.15.152:0/3519459672 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fa5500fbd00 0x7fa5500fc110 unknown :-1 s=CLOSED pgs=274 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:44.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.678+0000 7fa555b99700 1 -- 172.21.15.152:0/3519459672 >> 172.21.15.152:0/3519459672 conn(0x7fa5500f7480 msgr2=0x7fa5500f9890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:44.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.678+0000 7fa555b99700 1 -- 172.21.15.152:0/3519459672 shutdown_connections 2024-04-02T04:56:44.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.679+0000 7fa555b99700 1 -- 172.21.15.152:0/3519459672 wait complete. 2024-04-02T04:56:44.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.679+0000 7fa555b99700 1 Processor -- start 2024-04-02T04:56:44.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.679+0000 7fa555b99700 1 -- start start 2024-04-02T04:56:44.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.679+0000 7fa555b99700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa5500fbd00 0x7fa55010d980 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:44.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.679+0000 7fa555b99700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fa5500fcfb0 0x7fa55010dec0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:44.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.679+0000 7fa555b99700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa55010e550 con 0x7fa5500fcfb0 2024-04-02T04:56:44.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.679+0000 7fa555b99700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa550112ef0 con 0x7fa5500fbd00 2024-04-02T04:56:44.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.679+0000 7fa54effd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fa5500fcfb0 0x7fa55010dec0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:44.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.680+0000 7fa54effd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fa5500fcfb0 0x7fa55010dec0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:45160/0 (socket says 172.21.15.152:45160) 2024-04-02T04:56:44.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.680+0000 7fa54effd700 1 -- 172.21.15.152:0/4086178889 learned_addr learned my addr 172.21.15.152:0/4086178889 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:56:44.684 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.680+0000 7fa54f7fe700 1 --2- 172.21.15.152:0/4086178889 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa5500fbd00 0x7fa55010d980 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:44.685 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.680+0000 7fa54effd700 1 -- 172.21.15.152:0/4086178889 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa5500fbd00 msgr2=0x7fa55010d980 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:44.685 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.680+0000 7fa54effd700 1 --2- 172.21.15.152:0/4086178889 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa5500fbd00 0x7fa55010d980 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:44.685 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.680+0000 7fa54effd700 1 -- 172.21.15.152:0/4086178889 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa540005040 con 0x7fa5500fcfb0 2024-04-02T04:56:44.685 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.680+0000 7fa54effd700 1 --2- 172.21.15.152:0/4086178889 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fa5500fcfb0 0x7fa55010dec0 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7fa54400b790 tx=0x7fa544009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:44.685 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.680+0000 7fa537fff700 1 -- 172.21.15.152:0/4086178889 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fa54400f6f0 con 0x7fa5500fcfb0 2024-04-02T04:56:44.686 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.680+0000 7fa537fff700 1 -- 172.21.15.152:0/4086178889 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fa54400fd20 con 0x7fa5500fcfb0 2024-04-02T04:56:44.686 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.680+0000 7fa555b99700 1 -- 172.21.15.152:0/4086178889 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa550113150 con 0x7fa5500fcfb0 2024-04-02T04:56:44.686 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.680+0000 7fa537fff700 1 -- 172.21.15.152:0/4086178889 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fa5440103a0 con 0x7fa5500fcfb0 2024-04-02T04:56:44.686 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.681+0000 7fa555b99700 1 -- 172.21.15.152:0/4086178889 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa5501136c0 con 0x7fa5500fcfb0 2024-04-02T04:56:44.686 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.682+0000 7fa555b99700 1 -- 172.21.15.152:0/4086178889 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa550101b40 con 0x7fa5500fcfb0 2024-04-02T04:56:44.686 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.682+0000 7fa537fff700 1 -- 172.21.15.152:0/4086178889 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7fa544011400 con 0x7fa5500fcfb0 2024-04-02T04:56:44.687 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.682+0000 7fa537fff700 1 --2- 172.21.15.152:0/4086178889 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fa5380692a0 0x7fa53806b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:44.687 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.682+0000 7fa537fff700 1 -- 172.21.15.152:0/4086178889 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7fa544086fc0 con 0x7fa5500fcfb0 2024-04-02T04:56:44.687 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.683+0000 7fa54f7fe700 1 --2- 172.21.15.152:0/4086178889 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fa5380692a0 0x7fa53806b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:44.687 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.683+0000 7fa54f7fe700 1 --2- 172.21.15.152:0/4086178889 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fa5380692a0 0x7fa53806b750 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7fa540009fd0 tx=0x7fa540009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:44.690 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.687+0000 7fa537fff700 1 -- 172.21.15.152:0/4086178889 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fa544056d30 con 0x7fa5500fcfb0 2024-04-02T04:56:44.892 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.888+0000 7fa555b99700 1 -- 172.21.15.152:0/4086178889 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "target": ["mon-mgr", ""]}) v1 -- 0x7fa55002b310 con 0x7fa5380692a0 2024-04-02T04:56:44.897 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.894+0000 7fa537fff700 1 -- 172.21.15.152:0/4086178889 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+89 (secure 0 0 0) 0x7fa55002b310 con 0x7fa5380692a0 2024-04-02T04:56:44.897 INFO:teuthology.orchestra.run.smithi152.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T04:56:44.900 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.896+0000 7fa555b99700 1 -- 172.21.15.152:0/4086178889 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fa5380692a0 msgr2=0x7fa53806b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:44.900 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.896+0000 7fa555b99700 1 --2- 172.21.15.152:0/4086178889 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fa5380692a0 0x7fa53806b750 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 crypto rx=0x7fa540009fd0 tx=0x7fa540009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:44.900 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.896+0000 7fa555b99700 1 -- 172.21.15.152:0/4086178889 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fa5500fcfb0 msgr2=0x7fa55010dec0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:44.900 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.896+0000 7fa555b99700 1 --2- 172.21.15.152:0/4086178889 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fa5500fcfb0 0x7fa55010dec0 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 crypto rx=0x7fa54400b790 tx=0x7fa544009250 comp rx=0 tx=0).stop 2024-04-02T04:56:44.900 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.896+0000 7fa555b99700 1 -- 172.21.15.152:0/4086178889 shutdown_connections 2024-04-02T04:56:44.900 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.896+0000 7fa555b99700 1 --2- 172.21.15.152:0/4086178889 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fa5380692a0 0x7fa53806b750 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:44.900 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.896+0000 7fa555b99700 1 --2- 172.21.15.152:0/4086178889 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fa5500fbd00 0x7fa55010d980 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:44.900 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.896+0000 7fa555b99700 1 --2- 172.21.15.152:0/4086178889 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fa5500fcfb0 0x7fa55010dec0 unknown :-1 s=CLOSED pgs=275 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:44.900 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.896+0000 7fa555b99700 1 -- 172.21.15.152:0/4086178889 >> 172.21.15.152:0/4086178889 conn(0x7fa5500f7480 msgr2=0x7fa550066840 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:44.900 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.896+0000 7fa555b99700 1 -- 172.21.15.152:0/4086178889 shutdown_connections 2024-04-02T04:56:44.900 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:44.896+0000 7fa555b99700 1 -- 172.21.15.152:0/4086178889 wait complete. 2024-04-02T04:56:45.362 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:45 smithi152 ceph-mon[29073]: pgmap v201: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 3.0 KiB/s rd, 853 B/s wr, 5 op/s 2024-04-02T04:56:45.363 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:45 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:45.363 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:45 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:56:45.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:45 smithi154 ceph-mon[35729]: pgmap v201: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 3.0 KiB/s rd, 853 B/s wr, 5 op/s 2024-04-02T04:56:45.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:45 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:45.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:45 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:56:45.625 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-02T04:56:45.625 INFO:tasks.cephadm:Running commands on role host.a host ubuntu@smithi152.front.sepia.ceph.com 2024-04-02T04:56:45.626 DEBUG:teuthology.orchestra.run.smithi152:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2e0055d0-f0ac-11ee-b647-cb9ed24678a4 -e sha1=6cec793d9073f0594e2364b9b687ecaf22424cca -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph fs dump; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-04-02T04:56:45.909 INFO:teuthology.orchestra.run.smithi152.stderr:Inferring config /var/lib/ceph/2e0055d0-f0ac-11ee-b647-cb9ed24678a4/mon.smithi152/config 2024-04-02T04:56:46.313 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:46 smithi152 ceph-mon[29073]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:56:46.313 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:46 smithi152 ceph-mon[29073]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T04:56:46.313 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:46 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:56:46.313 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:46 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:56:46.313 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:46 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:46.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:46 smithi154 ceph-mon[35729]: from='client.14544 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:56:46.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:46 smithi154 ceph-mon[35729]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T04:56:46.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:46 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:56:46.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:46 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-02T04:56:46.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:46 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:56:47.312 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:47 smithi152 ceph-mon[29073]: pgmap v202: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 170 B/s wr, 3 op/s 2024-04-02T04:56:47.312 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:47 smithi152 ceph-mon[29073]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T04:56:47.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:47 smithi154 ceph-mon[35729]: pgmap v202: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 170 B/s wr, 3 op/s 2024-04-02T04:56:47.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:47 smithi154 ceph-mon[35729]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca 2024-04-02T04:56:47.556 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.541+0000 7fe7ad8c0700 1 -- 172.21.15.152:0/3554152979 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe7a80662e0 msgr2=0x7fe7a80ff9c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:47.556 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.541+0000 7fe7ad8c0700 1 --2- 172.21.15.152:0/3554152979 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe7a80662e0 0x7fe7a80ff9c0 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 crypto rx=0x7fe798004940 tx=0x7fe798009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:47.556 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.541+0000 7fe7ad8c0700 1 -- 172.21.15.152:0/3554152979 shutdown_connections 2024-04-02T04:56:47.556 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.541+0000 7fe7ad8c0700 1 --2- 172.21.15.152:0/3554152979 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe7a80fff00 0x7fe7a8102320 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:47.556 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.541+0000 7fe7ad8c0700 1 --2- 172.21.15.152:0/3554152979 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe7a80662e0 0x7fe7a80ff9c0 unknown :-1 s=CLOSED pgs=276 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:47.556 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.541+0000 7fe7ad8c0700 1 -- 172.21.15.152:0/3554152979 >> 172.21.15.152:0/3554152979 conn(0x7fe7a80f7430 msgr2=0x7fe7a80f9860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:47.556 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.542+0000 7fe7ad8c0700 1 -- 172.21.15.152:0/3554152979 shutdown_connections 2024-04-02T04:56:47.556 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.542+0000 7fe7ad8c0700 1 -- 172.21.15.152:0/3554152979 wait complete. 2024-04-02T04:56:47.556 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.542+0000 7fe7ad8c0700 1 Processor -- start 2024-04-02T04:56:47.556 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.542+0000 7fe7ad8c0700 1 -- start start 2024-04-02T04:56:47.557 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.542+0000 7fe7ad8c0700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe7a80662e0 0x7fe7a810da30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:47.557 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.542+0000 7fe7ad8c0700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe7a80fff00 0x7fe7a810df70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:47.557 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.542+0000 7fe7ad8c0700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe7a810e570 con 0x7fe7a80fff00 2024-04-02T04:56:47.557 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.542+0000 7fe7ad8c0700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe7a810edc0 con 0x7fe7a80662e0 2024-04-02T04:56:47.557 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.543+0000 7fe7a67fc700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe7a80fff00 0x7fe7a810df70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:47.557 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.543+0000 7fe7a67fc700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe7a80fff00 0x7fe7a810df70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:45178/0 (socket says 172.21.15.152:45178) 2024-04-02T04:56:47.557 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.543+0000 7fe7a67fc700 1 -- 172.21.15.152:0/1633093243 learned_addr learned my addr 172.21.15.152:0/1633093243 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:56:47.557 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.543+0000 7fe7a6ffd700 1 --2- 172.21.15.152:0/1633093243 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe7a80662e0 0x7fe7a810da30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:47.557 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.543+0000 7fe7a67fc700 1 -- 172.21.15.152:0/1633093243 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe7a80662e0 msgr2=0x7fe7a810da30 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:47.557 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.543+0000 7fe7a67fc700 1 --2- 172.21.15.152:0/1633093243 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe7a80662e0 0x7fe7a810da30 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:47.557 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.543+0000 7fe7a67fc700 1 -- 172.21.15.152:0/1633093243 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe798005040 con 0x7fe7a80fff00 2024-04-02T04:56:47.557 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.543+0000 7fe7a67fc700 1 --2- 172.21.15.152:0/1633093243 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe7a80fff00 0x7fe7a810df70 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7fe79c00c830 tx=0x7fe79c004470 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:47.557 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.543+0000 7fe78f7fe700 1 -- 172.21.15.152:0/1633093243 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fe79c0106f0 con 0x7fe7a80fff00 2024-04-02T04:56:47.557 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.543+0000 7fe7ad8c0700 1 -- 172.21.15.152:0/1633093243 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe7a810f0a0 con 0x7fe7a80fff00 2024-04-02T04:56:47.557 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.543+0000 7fe78f7fe700 1 -- 172.21.15.152:0/1633093243 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fe79c010d20 con 0x7fe7a80fff00 2024-04-02T04:56:47.558 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.543+0000 7fe78f7fe700 1 -- 172.21.15.152:0/1633093243 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fe79c0113a0 con 0x7fe7a80fff00 2024-04-02T04:56:47.558 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.543+0000 7fe7ad8c0700 1 -- 172.21.15.152:0/1633093243 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe7a810f5e0 con 0x7fe7a80fff00 2024-04-02T04:56:47.558 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.545+0000 7fe7ad8c0700 1 -- 172.21.15.152:0/1633093243 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe7a805f5f0 con 0x7fe7a80fff00 2024-04-02T04:56:47.558 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.545+0000 7fe78f7fe700 1 -- 172.21.15.152:0/1633093243 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7fe79c012400 con 0x7fe7a80fff00 2024-04-02T04:56:47.558 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.545+0000 7fe78f7fe700 1 --2- 172.21.15.152:0/1633093243 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fe7900692a0 0x7fe79006b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:47.558 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.545+0000 7fe78f7fe700 1 -- 172.21.15.152:0/1633093243 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7fe79c0871a0 con 0x7fe7a80fff00 2024-04-02T04:56:47.558 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.545+0000 7fe7a6ffd700 1 --2- 172.21.15.152:0/1633093243 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fe7900692a0 0x7fe79006b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:47.558 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.546+0000 7fe7a6ffd700 1 --2- 172.21.15.152:0/1633093243 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fe7900692a0 0x7fe79006b750 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7fe79800b970 tx=0x7fe798009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:47.558 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.550+0000 7fe78f7fe700 1 -- 172.21.15.152:0/1633093243 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fe79c056f10 con 0x7fe7a80fff00 2024-04-02T04:56:47.793 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.789+0000 7fe7ad8c0700 1 -- 172.21.15.152:0/1633093243 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}) v1 -- 0x7fe7a810f970 con 0x7fe7900692a0 2024-04-02T04:56:47.794 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.790+0000 7fe78f7fe700 1 -- 172.21.15.152:0/1633093243 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+358 (secure 0 0 0) 0x7fe7a810f970 con 0x7fe7900692a0 2024-04-02T04:56:47.797 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.792+0000 7fe7ad8c0700 1 -- 172.21.15.152:0/1633093243 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fe7900692a0 msgr2=0x7fe79006b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:47.797 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.792+0000 7fe7ad8c0700 1 --2- 172.21.15.152:0/1633093243 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fe7900692a0 0x7fe79006b750 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 crypto rx=0x7fe79800b970 tx=0x7fe798009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:47.797 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.792+0000 7fe7ad8c0700 1 -- 172.21.15.152:0/1633093243 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe7a80fff00 msgr2=0x7fe7a810df70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:47.797 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.792+0000 7fe7ad8c0700 1 --2- 172.21.15.152:0/1633093243 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe7a80fff00 0x7fe7a810df70 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 crypto rx=0x7fe79c00c830 tx=0x7fe79c004470 comp rx=0 tx=0).stop 2024-04-02T04:56:47.797 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.792+0000 7fe7ad8c0700 1 -- 172.21.15.152:0/1633093243 shutdown_connections 2024-04-02T04:56:47.797 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.792+0000 7fe7ad8c0700 1 --2- 172.21.15.152:0/1633093243 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fe7900692a0 0x7fe79006b750 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:47.797 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.792+0000 7fe7ad8c0700 1 --2- 172.21.15.152:0/1633093243 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe7a80662e0 0x7fe7a810da30 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:47.797 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.792+0000 7fe7ad8c0700 1 --2- 172.21.15.152:0/1633093243 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe7a80fff00 0x7fe7a810df70 unknown :-1 s=CLOSED pgs=277 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:47.797 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.792+0000 7fe7ad8c0700 1 -- 172.21.15.152:0/1633093243 >> 172.21.15.152:0/1633093243 conn(0x7fe7a80f7430 msgr2=0x7fe7a80f7850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:47.797 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.793+0000 7fe7ad8c0700 1 -- 172.21.15.152:0/1633093243 shutdown_connections 2024-04-02T04:56:47.797 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.793+0000 7fe7ad8c0700 1 -- 172.21.15.152:0/1633093243 wait complete. 2024-04-02T04:56:47.807 INFO:teuthology.orchestra.run.smithi152.stdout:true 2024-04-02T04:56:47.907 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.901+0000 7f12a8bb1700 1 -- 172.21.15.152:0/2218291610 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f12a40fcf80 msgr2=0x7f12a40fd3f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:47.907 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.901+0000 7f12a8bb1700 1 --2- 172.21.15.152:0/2218291610 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f12a40fcf80 0x7f12a40fd3f0 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 crypto rx=0x7f1298004940 tx=0x7f1298009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:47.907 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.902+0000 7f12a8bb1700 1 -- 172.21.15.152:0/2218291610 shutdown_connections 2024-04-02T04:56:47.907 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.902+0000 7f12a8bb1700 1 --2- 172.21.15.152:0/2218291610 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f12a40fcf80 0x7f12a40fd3f0 unknown :-1 s=CLOSED pgs=278 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:47.907 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.902+0000 7f12a8bb1700 1 --2- 172.21.15.152:0/2218291610 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f12a40fbcd0 0x7f12a40fc0e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:47.907 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.902+0000 7f12a8bb1700 1 -- 172.21.15.152:0/2218291610 >> 172.21.15.152:0/2218291610 conn(0x7f12a40f7430 msgr2=0x7f12a40f9860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:47.907 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.902+0000 7f12a8bb1700 1 -- 172.21.15.152:0/2218291610 shutdown_connections 2024-04-02T04:56:47.907 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.902+0000 7f12a8bb1700 1 -- 172.21.15.152:0/2218291610 wait complete. 2024-04-02T04:56:47.908 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.902+0000 7f12a8bb1700 1 Processor -- start 2024-04-02T04:56:47.908 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.902+0000 7f12a8bb1700 1 -- start start 2024-04-02T04:56:47.908 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.902+0000 7f12a8bb1700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f12a40fbcd0 0x7f12a40ffdf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:47.908 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.902+0000 7f12a8bb1700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f12a40fcf80 0x7f12a40fe2c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:47.908 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.902+0000 7f12a8bb1700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f12a40fe870 con 0x7f12a40fbcd0 2024-04-02T04:56:47.908 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.902+0000 7f12a8bb1700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f12a40fe9e0 con 0x7f12a40fcf80 2024-04-02T04:56:47.908 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.903+0000 7f12a259c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f12a40fbcd0 0x7f12a40ffdf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:47.908 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.903+0000 7f12a259c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f12a40fbcd0 0x7f12a40ffdf0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:45194/0 (socket says 172.21.15.152:45194) 2024-04-02T04:56:47.908 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.903+0000 7f12a259c700 1 -- 172.21.15.152:0/2091334439 learned_addr learned my addr 172.21.15.152:0/2091334439 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:56:47.908 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.903+0000 7f12a1d9b700 1 --2- 172.21.15.152:0/2091334439 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f12a40fcf80 0x7f12a40fe2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:47.908 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.903+0000 7f12a259c700 1 -- 172.21.15.152:0/2091334439 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f12a40fcf80 msgr2=0x7f12a40fe2c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:47.908 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.903+0000 7f12a259c700 1 --2- 172.21.15.152:0/2091334439 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f12a40fcf80 0x7f12a40fe2c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:47.908 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.903+0000 7f12a259c700 1 -- 172.21.15.152:0/2091334439 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1298005040 con 0x7f12a40fbcd0 2024-04-02T04:56:47.908 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.903+0000 7f12a1d9b700 1 --2- 172.21.15.152:0/2091334439 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f12a40fcf80 0x7f12a40fe2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_reply_more state changed! 2024-04-02T04:56:47.909 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.903+0000 7f12a259c700 1 --2- 172.21.15.152:0/2091334439 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f12a40fbcd0 0x7f12a40ffdf0 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7f1294002a50 tx=0x7f12940046b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:47.909 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.903+0000 7f1292ffd700 1 -- 172.21.15.152:0/2091334439 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f1294007cc0 con 0x7f12a40fbcd0 2024-04-02T04:56:47.909 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.903+0000 7f12a8bb1700 1 -- 172.21.15.152:0/2091334439 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f12a40fecc0 con 0x7f12a40fbcd0 2024-04-02T04:56:47.909 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.903+0000 7f1292ffd700 1 -- 172.21.15.152:0/2091334439 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f129400f040 con 0x7f12a40fbcd0 2024-04-02T04:56:47.909 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.903+0000 7f1292ffd700 1 -- 172.21.15.152:0/2091334439 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f1294009640 con 0x7f12a40fbcd0 2024-04-02T04:56:47.910 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.904+0000 7f12a8bb1700 1 -- 172.21.15.152:0/2091334439 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f12a4115a40 con 0x7f12a40fbcd0 2024-04-02T04:56:47.910 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.905+0000 7f12a8bb1700 1 -- 172.21.15.152:0/2091334439 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f12a405f5f0 con 0x7f12a40fbcd0 2024-04-02T04:56:47.910 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.905+0000 7f1292ffd700 1 -- 172.21.15.152:0/2091334439 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f12940097a0 con 0x7f12a40fbcd0 2024-04-02T04:56:47.910 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.905+0000 7f1292ffd700 1 --2- 172.21.15.152:0/2091334439 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f128c0692a0 0x7f128c06b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:47.910 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.905+0000 7f1292ffd700 1 -- 172.21.15.152:0/2091334439 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7f1294086ca0 con 0x7f12a40fbcd0 2024-04-02T04:56:47.910 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.905+0000 7f12a1d9b700 1 --2- 172.21.15.152:0/2091334439 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f128c0692a0 0x7f128c06b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:47.910 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.906+0000 7f12a1d9b700 1 --2- 172.21.15.152:0/2091334439 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f128c0692a0 0x7f128c06b750 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f129800b3b0 tx=0x7f1298009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:47.913 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:47.910+0000 7f1292ffd700 1 -- 172.21.15.152:0/2091334439 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f129408b070 con 0x7f12a40fbcd0 2024-04-02T04:56:48.150 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.146+0000 7f12a8bb1700 1 -- 172.21.15.152:0/2091334439 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}) v1 -- 0x7f12a405e400 con 0x7f128c0692a0 2024-04-02T04:56:48.150 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.147+0000 7f1292ffd700 1 -- 172.21.15.152:0/2091334439 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+358 (secure 0 0 0) 0x7f12a405e400 con 0x7f128c0692a0 2024-04-02T04:56:48.153 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.149+0000 7f12a8bb1700 1 -- 172.21.15.152:0/2091334439 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f128c0692a0 msgr2=0x7f128c06b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:48.153 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.149+0000 7f12a8bb1700 1 --2- 172.21.15.152:0/2091334439 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f128c0692a0 0x7f128c06b750 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 crypto rx=0x7f129800b3b0 tx=0x7f1298009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:48.153 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.149+0000 7f12a8bb1700 1 -- 172.21.15.152:0/2091334439 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f12a40fbcd0 msgr2=0x7f12a40ffdf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:48.153 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.149+0000 7f12a8bb1700 1 --2- 172.21.15.152:0/2091334439 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f12a40fbcd0 0x7f12a40ffdf0 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 crypto rx=0x7f1294002a50 tx=0x7f12940046b0 comp rx=0 tx=0).stop 2024-04-02T04:56:48.153 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.149+0000 7f12a8bb1700 1 -- 172.21.15.152:0/2091334439 shutdown_connections 2024-04-02T04:56:48.153 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.149+0000 7f12a8bb1700 1 --2- 172.21.15.152:0/2091334439 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f128c0692a0 0x7f128c06b750 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:48.153 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.149+0000 7f12a8bb1700 1 --2- 172.21.15.152:0/2091334439 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f12a40fbcd0 0x7f12a40ffdf0 unknown :-1 s=CLOSED pgs=279 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:48.153 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.149+0000 7f12a8bb1700 1 --2- 172.21.15.152:0/2091334439 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f12a40fcf80 0x7f12a40fe2c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:48.153 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.149+0000 7f12a8bb1700 1 -- 172.21.15.152:0/2091334439 >> 172.21.15.152:0/2091334439 conn(0x7f12a40f7430 msgr2=0x7f12a4065b00 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:48.154 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.149+0000 7f12a8bb1700 1 -- 172.21.15.152:0/2091334439 shutdown_connections 2024-04-02T04:56:48.154 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.149+0000 7f12a8bb1700 1 -- 172.21.15.152:0/2091334439 wait complete. 2024-04-02T04:56:48.260 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.256+0000 7f80ac102700 1 -- 172.21.15.152:0/311982613 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f80a40fce80 msgr2=0x7f80a40fd290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:48.260 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.256+0000 7f80ac102700 1 --2- 172.21.15.152:0/311982613 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f80a40fce80 0x7f80a40fd290 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 crypto rx=0x7f80a0004940 tx=0x7f80a0009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:48.260 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.256+0000 7f80ac102700 1 -- 172.21.15.152:0/311982613 shutdown_connections 2024-04-02T04:56:48.260 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.256+0000 7f80ac102700 1 --2- 172.21.15.152:0/311982613 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f80a40fdff0 0x7f80a40fe460 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:48.260 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.256+0000 7f80ac102700 1 --2- 172.21.15.152:0/311982613 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f80a40fce80 0x7f80a40fd290 unknown :-1 s=CLOSED pgs=280 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:48.260 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.256+0000 7f80ac102700 1 -- 172.21.15.152:0/311982613 >> 172.21.15.152:0/311982613 conn(0x7f80a40f8540 msgr2=0x7f80a40fa970 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:48.260 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.256+0000 7f80ac102700 1 -- 172.21.15.152:0/311982613 shutdown_connections 2024-04-02T04:56:48.260 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.256+0000 7f80ac102700 1 -- 172.21.15.152:0/311982613 wait complete. 2024-04-02T04:56:48.262 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.257+0000 7f80ac102700 1 Processor -- start 2024-04-02T04:56:48.262 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.257+0000 7f80ac102700 1 -- start start 2024-04-02T04:56:48.262 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.257+0000 7f80ac102700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f80a40fce80 0x7f80a4071e40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:48.262 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.257+0000 7f80ac102700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f80a40fdff0 0x7f80a4070310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:48.262 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.257+0000 7f80ac102700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80a4070950 con 0x7f80a40fdff0 2024-04-02T04:56:48.262 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.257+0000 7f80ac102700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f80a4070ac0 con 0x7f80a40fce80 2024-04-02T04:56:48.262 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.257+0000 7f80a969d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f80a40fdff0 0x7f80a4070310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:48.262 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.257+0000 7f80a969d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f80a40fdff0 0x7f80a4070310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:45208/0 (socket says 172.21.15.152:45208) 2024-04-02T04:56:48.262 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.257+0000 7f80a969d700 1 -- 172.21.15.152:0/604476766 learned_addr learned my addr 172.21.15.152:0/604476766 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:56:48.262 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.257+0000 7f80a9e9e700 1 --2- 172.21.15.152:0/604476766 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f80a40fce80 0x7f80a4071e40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:48.262 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.257+0000 7f80a969d700 1 -- 172.21.15.152:0/604476766 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f80a40fce80 msgr2=0x7f80a4071e40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:48.263 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.257+0000 7f80a969d700 1 --2- 172.21.15.152:0/604476766 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f80a40fce80 0x7f80a4071e40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:48.263 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.257+0000 7f80a969d700 1 -- 172.21.15.152:0/604476766 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f80a0005040 con 0x7f80a40fdff0 2024-04-02T04:56:48.263 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.257+0000 7f80a969d700 1 --2- 172.21.15.152:0/604476766 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f80a40fdff0 0x7f80a4070310 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7f809400b790 tx=0x7f8094009250 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:48.263 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.258+0000 7f809a7fc700 1 -- 172.21.15.152:0/604476766 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f809400f6f0 con 0x7f80a40fdff0 2024-04-02T04:56:48.263 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.258+0000 7f809a7fc700 1 -- 172.21.15.152:0/604476766 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f809400fd20 con 0x7f80a40fdff0 2024-04-02T04:56:48.263 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.258+0000 7f80ac102700 1 -- 172.21.15.152:0/604476766 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f80a4070da0 con 0x7f80a40fdff0 2024-04-02T04:56:48.263 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.258+0000 7f809a7fc700 1 -- 172.21.15.152:0/604476766 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f80940103a0 con 0x7f80a40fdff0 2024-04-02T04:56:48.263 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.258+0000 7f80ac102700 1 -- 172.21.15.152:0/604476766 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f80a4115ae0 con 0x7f80a40fdff0 2024-04-02T04:56:48.264 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.259+0000 7f80ac102700 1 -- 172.21.15.152:0/604476766 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f80a405f5f0 con 0x7f80a40fdff0 2024-04-02T04:56:48.264 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.259+0000 7f809a7fc700 1 -- 172.21.15.152:0/604476766 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f809400f850 con 0x7f80a40fdff0 2024-04-02T04:56:48.264 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.260+0000 7f809a7fc700 1 --2- 172.21.15.152:0/604476766 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f80900692a0 0x7f809006b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:48.264 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.260+0000 7f809a7fc700 1 -- 172.21.15.152:0/604476766 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7f8094086f00 con 0x7f80a40fdff0 2024-04-02T04:56:48.264 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.260+0000 7f80a9e9e700 1 --2- 172.21.15.152:0/604476766 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f80900692a0 0x7f809006b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:48.264 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.260+0000 7f80a9e9e700 1 --2- 172.21.15.152:0/604476766 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f80900692a0 0x7f809006b750 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f80a0004890 tx=0x7f80a0009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:48.268 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.264+0000 7f809a7fc700 1 -- 172.21.15.152:0/604476766 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f8094053280 con 0x7f80a40fdff0 2024-04-02T04:56:48.478 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.474+0000 7f80ac102700 1 -- 172.21.15.152:0/604476766 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "orch ps", "target": ["mon-mgr", ""]}) v1 -- 0x7f80a4102290 con 0x7f80900692a0 2024-04-02T04:56:48.488 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.484+0000 7f809a7fc700 1 -- 172.21.15.152:0/604476766 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+3366 (secure 0 0 0) 0x7f80a4102290 con 0x7f80900692a0 2024-04-02T04:56:48.488 INFO:teuthology.orchestra.run.smithi152.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T04:56:48.488 INFO:teuthology.orchestra.run.smithi152.stdout:alertmanager.smithi152 smithi152 *:9093,9094 running (4m) 28s ago 6m 25.2M - 0.25.0 c8568f914cd2 1f52e384342b 2024-04-02T04:56:48.489 INFO:teuthology.orchestra.run.smithi152.stdout:crash.smithi152 smithi152 running (6m) 28s ago 6m 6665k - 17.2.7-803-gc924f991 4948510fac0f 12f8a8489975 2024-04-02T04:56:48.489 INFO:teuthology.orchestra.run.smithi152.stdout:crash.smithi154 smithi154 running (5m) 35s ago 5m 6665k - 17.2.7-803-gc924f991 4948510fac0f 7fe215d49e0f 2024-04-02T04:56:48.489 INFO:teuthology.orchestra.run.smithi152.stdout:grafana.smithi152 smithi152 *:3000 running (4m) 28s ago 6m 85.7M - 9.4.7 954c08fa6188 eb6036b14f99 2024-04-02T04:56:48.489 INFO:teuthology.orchestra.run.smithi152.stdout:mds.cephfs.smithi152.lluqrs smithi152 running (55s) 28s ago 54s 15.0M - 17.2.7-803-gc924f991 4948510fac0f bada96a57e65 2024-04-02T04:56:48.489 INFO:teuthology.orchestra.run.smithi152.stdout:mds.cephfs.smithi152.ywghzy smithi152 running (48s) 28s ago 48s 15.9M - 17.2.7-803-gc924f991 4948510fac0f 267b6d51cad3 2024-04-02T04:56:48.489 INFO:teuthology.orchestra.run.smithi152.stdout:mds.cephfs.smithi154.llvaoh smithi154 running (52s) 35s ago 51s 11.3M - 17.2.7-803-gc924f991 4948510fac0f 82f2bed61253 2024-04-02T04:56:48.489 INFO:teuthology.orchestra.run.smithi152.stdout:mds.cephfs.smithi154.rbkmwp smithi154 running (46s) 35s ago 45s 14.7M - 17.2.7-803-gc924f991 4948510fac0f eba3a7f41a09 2024-04-02T04:56:48.489 INFO:teuthology.orchestra.run.smithi152.stdout:mgr.smithi152.jivdcl smithi152 *:9283 running (7m) 28s ago 7m 481M - 17.2.7-803-gc924f991 4948510fac0f 2fd5fbea3f34 2024-04-02T04:56:48.489 INFO:teuthology.orchestra.run.smithi152.stdout:mgr.smithi154.pplnxz smithi154 *:8443,9283 running (5m) 35s ago 5m 432M - 17.2.7-803-gc924f991 4948510fac0f 0a1df688b4ac 2024-04-02T04:56:48.489 INFO:teuthology.orchestra.run.smithi152.stdout:mon.smithi152 smithi152 running (7m) 28s ago 7m 46.3M 2048M 17.2.7-803-gc924f991 4948510fac0f 399b9cc778f0 2024-04-02T04:56:48.489 INFO:teuthology.orchestra.run.smithi152.stdout:mon.smithi154 smithi154 running (5m) 35s ago 5m 36.0M 2048M 17.2.7-803-gc924f991 4948510fac0f 975aefa2215f 2024-04-02T04:56:48.489 INFO:teuthology.orchestra.run.smithi152.stdout:node-exporter.smithi152 smithi152 *:9100 running (6m) 28s ago 6m 16.5M - 1.5.0 0da6a335fe13 8168d9a53e30 2024-04-02T04:56:48.489 INFO:teuthology.orchestra.run.smithi152.stdout:node-exporter.smithi154 smithi154 *:9100 running (5m) 35s ago 5m 15.7M - 1.5.0 0da6a335fe13 8ee2f7610052 2024-04-02T04:56:48.489 INFO:teuthology.orchestra.run.smithi152.stdout:osd.0 smithi152 running (4m) 28s ago 4m 45.0M 4288M 17.2.7-803-gc924f991 4948510fac0f 9935e01f3199 2024-04-02T04:56:48.490 INFO:teuthology.orchestra.run.smithi152.stdout:osd.1 smithi152 running (4m) 28s ago 4m 46.5M 4288M 17.2.7-803-gc924f991 4948510fac0f e5c49d238944 2024-04-02T04:56:48.490 INFO:teuthology.orchestra.run.smithi152.stdout:osd.2 smithi152 running (3m) 28s ago 3m 47.1M 4288M 17.2.7-803-gc924f991 4948510fac0f c53a93ae7737 2024-04-02T04:56:48.490 INFO:teuthology.orchestra.run.smithi152.stdout:osd.3 smithi154 running (3m) 35s ago 3m 44.8M 5312M 17.2.7-803-gc924f991 4948510fac0f 959ebaab3713 2024-04-02T04:56:48.490 INFO:teuthology.orchestra.run.smithi152.stdout:osd.4 smithi154 running (2m) 35s ago 2m 41.7M 5312M 17.2.7-803-gc924f991 4948510fac0f 4856c813e7fd 2024-04-02T04:56:48.490 INFO:teuthology.orchestra.run.smithi152.stdout:osd.5 smithi154 running (2m) 35s ago 2m 42.0M 5312M 17.2.7-803-gc924f991 4948510fac0f fbc6bb91d1a7 2024-04-02T04:56:48.490 INFO:teuthology.orchestra.run.smithi152.stdout:prometheus.smithi152 smithi152 *:9095 running (4m) 28s ago 5m 53.8M - 2.43.0 a07b618ecd1d 874955f0a102 2024-04-02T04:56:48.491 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.487+0000 7f80ac102700 1 -- 172.21.15.152:0/604476766 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f80900692a0 msgr2=0x7f809006b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:48.491 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.487+0000 7f80ac102700 1 --2- 172.21.15.152:0/604476766 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f80900692a0 0x7f809006b750 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 crypto rx=0x7f80a0004890 tx=0x7f80a0009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:48.491 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.487+0000 7f80ac102700 1 -- 172.21.15.152:0/604476766 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f80a40fdff0 msgr2=0x7f80a4070310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:48.491 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.487+0000 7f80ac102700 1 --2- 172.21.15.152:0/604476766 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f80a40fdff0 0x7f80a4070310 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 crypto rx=0x7f809400b790 tx=0x7f8094009250 comp rx=0 tx=0).stop 2024-04-02T04:56:48.491 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.487+0000 7f80ac102700 1 -- 172.21.15.152:0/604476766 shutdown_connections 2024-04-02T04:56:48.491 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.487+0000 7f80ac102700 1 --2- 172.21.15.152:0/604476766 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f80900692a0 0x7f809006b750 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:48.491 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.487+0000 7f80ac102700 1 --2- 172.21.15.152:0/604476766 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f80a40fce80 0x7f80a4071e40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:48.491 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.487+0000 7f80ac102700 1 --2- 172.21.15.152:0/604476766 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f80a40fdff0 0x7f80a4070310 unknown :-1 s=CLOSED pgs=281 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:48.491 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.487+0000 7f80ac102700 1 -- 172.21.15.152:0/604476766 >> 172.21.15.152:0/604476766 conn(0x7f80a40f8540 msgr2=0x7f80a40f9b90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:48.491 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.487+0000 7f80ac102700 1 -- 172.21.15.152:0/604476766 shutdown_connections 2024-04-02T04:56:48.492 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.487+0000 7f80ac102700 1 -- 172.21.15.152:0/604476766 wait complete. 2024-04-02T04:56:48.599 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.594+0000 7fdc23450700 1 -- 172.21.15.152:0/2447237884 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fdc1c0fe060 msgr2=0x7fdc1c0fe4d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:48.599 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.594+0000 7fdc23450700 1 --2- 172.21.15.152:0/2447237884 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fdc1c0fe060 0x7fdc1c0fe4d0 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 crypto rx=0x7fdc0c004940 tx=0x7fdc0c009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:48.599 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.594+0000 7fdc23450700 1 -- 172.21.15.152:0/2447237884 shutdown_connections 2024-04-02T04:56:48.599 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.594+0000 7fdc23450700 1 --2- 172.21.15.152:0/2447237884 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fdc1c0fe060 0x7fdc1c0fe4d0 unknown :-1 s=CLOSED pgs=282 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:48.599 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.594+0000 7fdc23450700 1 --2- 172.21.15.152:0/2447237884 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdc1c0fcef0 0x7fdc1c0fd300 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:48.599 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.594+0000 7fdc23450700 1 -- 172.21.15.152:0/2447237884 >> 172.21.15.152:0/2447237884 conn(0x7fdc1c0f85b0 msgr2=0x7fdc1c0fa9e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:48.599 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.594+0000 7fdc23450700 1 -- 172.21.15.152:0/2447237884 shutdown_connections 2024-04-02T04:56:48.599 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.594+0000 7fdc23450700 1 -- 172.21.15.152:0/2447237884 wait complete. 2024-04-02T04:56:48.599 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.595+0000 7fdc23450700 1 Processor -- start 2024-04-02T04:56:48.599 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.595+0000 7fdc23450700 1 -- start start 2024-04-02T04:56:48.600 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.595+0000 7fdc23450700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdc1c0fcef0 0x7fdc1c10da40 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:48.600 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.595+0000 7fdc23450700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fdc1c0fe060 0x7fdc1c10df80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:48.600 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.595+0000 7fdc23450700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdc1c10e580 con 0x7fdc1c0fe060 2024-04-02T04:56:48.600 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.595+0000 7fdc23450700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdc1c10edd0 con 0x7fdc1c0fcef0 2024-04-02T04:56:48.601 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.595+0000 7fdc209eb700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fdc1c0fe060 0x7fdc1c10df80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:48.601 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.595+0000 7fdc209eb700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fdc1c0fe060 0x7fdc1c10df80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:45228/0 (socket says 172.21.15.152:45228) 2024-04-02T04:56:48.601 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.595+0000 7fdc209eb700 1 -- 172.21.15.152:0/679680099 learned_addr learned my addr 172.21.15.152:0/679680099 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:56:48.601 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.596+0000 7fdc211ec700 1 --2- 172.21.15.152:0/679680099 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdc1c0fcef0 0x7fdc1c10da40 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:48.601 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.596+0000 7fdc209eb700 1 -- 172.21.15.152:0/679680099 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdc1c0fcef0 msgr2=0x7fdc1c10da40 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:48.601 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.596+0000 7fdc209eb700 1 --2- 172.21.15.152:0/679680099 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdc1c0fcef0 0x7fdc1c10da40 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:48.601 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.596+0000 7fdc209eb700 1 -- 172.21.15.152:0/679680099 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdc0c005040 con 0x7fdc1c0fe060 2024-04-02T04:56:48.601 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.596+0000 7fdc209eb700 1 --2- 172.21.15.152:0/679680099 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fdc1c0fe060 0x7fdc1c10df80 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7fdc0c009150 tx=0x7fdc0c009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:48.601 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.596+0000 7fdc11ffb700 1 -- 172.21.15.152:0/679680099 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fdc0c00e040 con 0x7fdc1c0fe060 2024-04-02T04:56:48.601 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.596+0000 7fdc11ffb700 1 -- 172.21.15.152:0/679680099 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fdc0c004290 con 0x7fdc1c0fe060 2024-04-02T04:56:48.602 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.596+0000 7fdc23450700 1 -- 172.21.15.152:0/679680099 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdc1c10f050 con 0x7fdc1c0fe060 2024-04-02T04:56:48.602 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.596+0000 7fdc11ffb700 1 -- 172.21.15.152:0/679680099 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fdc0c01c630 con 0x7fdc1c0fe060 2024-04-02T04:56:48.602 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.596+0000 7fdc23450700 1 -- 172.21.15.152:0/679680099 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdc1c10f590 con 0x7fdc1c0fe060 2024-04-02T04:56:48.603 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.598+0000 7fdc23450700 1 -- 172.21.15.152:0/679680099 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdc1c05f5f0 con 0x7fdc1c0fe060 2024-04-02T04:56:48.603 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.598+0000 7fdc11ffb700 1 -- 172.21.15.152:0/679680099 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7fdc0c00bbf0 con 0x7fdc1c0fe060 2024-04-02T04:56:48.603 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.598+0000 7fdc11ffb700 1 --2- 172.21.15.152:0/679680099 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fdc080692a0 0x7fdc0806b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:48.603 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.598+0000 7fdc11ffb700 1 -- 172.21.15.152:0/679680099 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7fdc0c087eb0 con 0x7fdc1c0fe060 2024-04-02T04:56:48.603 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.598+0000 7fdc211ec700 1 --2- 172.21.15.152:0/679680099 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fdc080692a0 0x7fdc0806b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:48.603 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.599+0000 7fdc211ec700 1 --2- 172.21.15.152:0/679680099 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fdc080692a0 0x7fdc0806b750 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7fdc18007fd0 tx=0x7fdc18009040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:48.606 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.603+0000 7fdc11ffb700 1 -- 172.21.15.152:0/679680099 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fdc0c057c20 con 0x7fdc1c0fe060 2024-04-02T04:56:48.901 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.897+0000 7fdc23450700 1 -- 172.21.15.152:0/679680099 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "versions"} v 0) v1 -- 0x7fdc1c10f870 con 0x7fdc1c0fe060 2024-04-02T04:56:48.902 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.898+0000 7fdc11ffb700 1 -- 172.21.15.152:0/679680099 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "versions"}]=0 v0) v1 ==== 56+0+638 (secure 0 0 0) 0x7fdc0c0577b0 con 0x7fdc1c0fe060 2024-04-02T04:56:48.902 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-04-02T04:56:48.902 INFO:teuthology.orchestra.run.smithi152.stdout: "mon": { 2024-04-02T04:56:48.902 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 17.2.7-803-gc924f991 (c924f9914f2809997b363f77c5cbb14929881220) quincy (stable)": 2 2024-04-02T04:56:48.902 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-04-02T04:56:48.902 INFO:teuthology.orchestra.run.smithi152.stdout: "mgr": { 2024-04-02T04:56:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 17.2.7-803-gc924f991 (c924f9914f2809997b363f77c5cbb14929881220) quincy (stable)": 2 2024-04-02T04:56:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-04-02T04:56:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: "osd": { 2024-04-02T04:56:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 17.2.7-803-gc924f991 (c924f9914f2809997b363f77c5cbb14929881220) quincy (stable)": 6 2024-04-02T04:56:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-04-02T04:56:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: "mds": { 2024-04-02T04:56:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 17.2.7-803-gc924f991 (c924f9914f2809997b363f77c5cbb14929881220) quincy (stable)": 4 2024-04-02T04:56:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-04-02T04:56:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: "overall": { 2024-04-02T04:56:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 17.2.7-803-gc924f991 (c924f9914f2809997b363f77c5cbb14929881220) quincy (stable)": 14 2024-04-02T04:56:48.903 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-04-02T04:56:48.903 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-04-02T04:56:48.904 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.900+0000 7fdc23450700 1 -- 172.21.15.152:0/679680099 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fdc080692a0 msgr2=0x7fdc0806b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:48.904 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.900+0000 7fdc23450700 1 --2- 172.21.15.152:0/679680099 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fdc080692a0 0x7fdc0806b750 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 crypto rx=0x7fdc18007fd0 tx=0x7fdc18009040 comp rx=0 tx=0).stop 2024-04-02T04:56:48.904 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.900+0000 7fdc23450700 1 -- 172.21.15.152:0/679680099 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fdc1c0fe060 msgr2=0x7fdc1c10df80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:48.905 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.900+0000 7fdc23450700 1 --2- 172.21.15.152:0/679680099 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fdc1c0fe060 0x7fdc1c10df80 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 crypto rx=0x7fdc0c009150 tx=0x7fdc0c009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:48.905 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.900+0000 7fdc23450700 1 -- 172.21.15.152:0/679680099 shutdown_connections 2024-04-02T04:56:48.905 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.900+0000 7fdc23450700 1 --2- 172.21.15.152:0/679680099 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fdc080692a0 0x7fdc0806b750 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:48.905 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.900+0000 7fdc23450700 1 --2- 172.21.15.152:0/679680099 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fdc1c0fcef0 0x7fdc1c10da40 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:48.905 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.900+0000 7fdc23450700 1 --2- 172.21.15.152:0/679680099 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fdc1c0fe060 0x7fdc1c10df80 unknown :-1 s=CLOSED pgs=283 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:48.905 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.900+0000 7fdc23450700 1 -- 172.21.15.152:0/679680099 >> 172.21.15.152:0/679680099 conn(0x7fdc1c0f85b0 msgr2=0x7fdc1c0f9c90 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:48.905 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.900+0000 7fdc23450700 1 -- 172.21.15.152:0/679680099 shutdown_connections 2024-04-02T04:56:48.905 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:48.900+0000 7fdc23450700 1 -- 172.21.15.152:0/679680099 wait complete. 2024-04-02T04:56:49.014 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.010+0000 7feadcd22700 1 -- 172.21.15.152:0/4162534028 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fead80fee10 msgr2=0x7fead80faf50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:49.014 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.010+0000 7feadcd22700 1 --2- 172.21.15.152:0/4162534028 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fead80fee10 0x7fead80faf50 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 crypto rx=0x7feac0004940 tx=0x7feac0009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:49.014 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.010+0000 7feadcd22700 1 -- 172.21.15.152:0/4162534028 shutdown_connections 2024-04-02T04:56:49.014 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.010+0000 7feadcd22700 1 --2- 172.21.15.152:0/4162534028 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fead80fee10 0x7fead80faf50 unknown :-1 s=CLOSED pgs=284 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:49.014 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.010+0000 7feadcd22700 1 --2- 172.21.15.152:0/4162534028 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fead80fe4c0 0x7fead80fe8d0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:49.015 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.010+0000 7feadcd22700 1 -- 172.21.15.152:0/4162534028 >> 172.21.15.152:0/4162534028 conn(0x7fead80f6ba0 msgr2=0x7fead80f8fd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:49.015 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.010+0000 7feadcd22700 1 -- 172.21.15.152:0/4162534028 shutdown_connections 2024-04-02T04:56:49.015 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.010+0000 7feadcd22700 1 -- 172.21.15.152:0/4162534028 wait complete. 2024-04-02T04:56:49.015 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.010+0000 7feadcd22700 1 Processor -- start 2024-04-02T04:56:49.015 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.010+0000 7feadcd22700 1 -- start start 2024-04-02T04:56:49.015 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.011+0000 7feadcd22700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fead80fe4c0 0x7fead810da30 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:49.015 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.011+0000 7feadcd22700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fead80fee10 0x7fead810df70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:49.015 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.011+0000 7feadcd22700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fead810e570 con 0x7fead80fe4c0 2024-04-02T04:56:49.015 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.011+0000 7feadcd22700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fead810edc0 con 0x7fead80fee10 2024-04-02T04:56:49.016 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.011+0000 7fead659c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fead80fe4c0 0x7fead810da30 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:49.016 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.011+0000 7fead659c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fead80fe4c0 0x7fead810da30 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:45236/0 (socket says 172.21.15.152:45236) 2024-04-02T04:56:49.016 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.011+0000 7fead659c700 1 -- 172.21.15.152:0/1589568443 learned_addr learned my addr 172.21.15.152:0/1589568443 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:56:49.016 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.011+0000 7fead659c700 1 -- 172.21.15.152:0/1589568443 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fead80fee10 msgr2=0x7fead810df70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:49.016 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.011+0000 7fead659c700 1 --2- 172.21.15.152:0/1589568443 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fead80fee10 0x7fead810df70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:49.016 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.011+0000 7fead659c700 1 -- 172.21.15.152:0/1589568443 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feac0005040 con 0x7fead80fe4c0 2024-04-02T04:56:49.017 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.011+0000 7fead659c700 1 --2- 172.21.15.152:0/1589568443 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fead80fe4c0 0x7fead810da30 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7feac800c830 tx=0x7feac8004470 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:49.017 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.011+0000 7feaceffd700 1 -- 172.21.15.152:0/1589568443 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7feac800a760 con 0x7fead80fe4c0 2024-04-02T04:56:49.017 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.011+0000 7feadcd22700 1 -- 172.21.15.152:0/1589568443 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fead810f0a0 con 0x7fead80fe4c0 2024-04-02T04:56:49.017 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.011+0000 7feaceffd700 1 -- 172.21.15.152:0/1589568443 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7feac800ad90 con 0x7fead80fe4c0 2024-04-02T04:56:49.017 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.011+0000 7feaceffd700 1 -- 172.21.15.152:0/1589568443 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7feac80113a0 con 0x7fead80fe4c0 2024-04-02T04:56:49.017 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.012+0000 7feadcd22700 1 -- 172.21.15.152:0/1589568443 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fead810f5e0 con 0x7fead80fe4c0 2024-04-02T04:56:49.017 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.013+0000 7feadcd22700 1 -- 172.21.15.152:0/1589568443 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fead805f5f0 con 0x7fead80fe4c0 2024-04-02T04:56:49.018 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.013+0000 7feaceffd700 1 -- 172.21.15.152:0/1589568443 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7feac8011610 con 0x7fead80fe4c0 2024-04-02T04:56:49.018 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.013+0000 7feaceffd700 1 --2- 172.21.15.152:0/1589568443 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7feac40692a0 0x7feac406b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:49.018 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.014+0000 7feaceffd700 1 -- 172.21.15.152:0/1589568443 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7feac8087270 con 0x7fead80fe4c0 2024-04-02T04:56:49.018 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.014+0000 7feacffff700 1 --2- 172.21.15.152:0/1589568443 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7feac40692a0 0x7feac406b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:49.018 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.014+0000 7feacffff700 1 --2- 172.21.15.152:0/1589568443 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7feac40692a0 0x7feac406b750 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7feac0004f90 tx=0x7feac0009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:49.021 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.018+0000 7feaceffd700 1 -- 172.21.15.152:0/1589568443 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7feac8057050 con 0x7fead80fe4c0 2024-04-02T04:56:49.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:49 smithi152 ceph-mon[29073]: from='client.14548 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:56:49.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:49 smithi152 ceph-mon[29073]: pgmap v203: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 170 B/s wr, 4 op/s 2024-04-02T04:56:49.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:49 smithi152 ceph-mon[29073]: from='client.14552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:56:49.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:49 smithi152 ceph-mon[29073]: from='client.14556 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:56:49.257 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:49 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/679680099' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T04:56:49.276 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.272+0000 7feadcd22700 1 -- 172.21.15.152:0/1589568443 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "fs dump"} v 0) v1 -- 0x7fead804bed0 con 0x7fead80fe4c0 2024-04-02T04:56:49.276 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.273+0000 7feaceffd700 1 -- 172.21.15.152:0/1589568443 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump"}]=0 dumped fsmap epoch 20 v20) v1 ==== 76+0+1833 (secure 0 0 0) 0x7feac8056be0 con 0x7fead80fe4c0 2024-04-02T04:56:49.277 INFO:teuthology.orchestra.run.smithi152.stdout:e20 2024-04-02T04:56:49.277 INFO:teuthology.orchestra.run.smithi152.stdout:enable_multiple, ever_enabled_multiple: 1,1 2024-04-02T04:56:49.277 INFO:teuthology.orchestra.run.smithi152.stdout:default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-02T04:56:49.277 INFO:teuthology.orchestra.run.smithi152.stdout:legacy client fscid: 1 2024-04-02T04:56:49.277 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:56:49.277 INFO:teuthology.orchestra.run.smithi152.stdout:Filesystem 'cephfs' (1) 2024-04-02T04:56:49.277 INFO:teuthology.orchestra.run.smithi152.stdout:fs_name cephfs 2024-04-02T04:56:49.277 INFO:teuthology.orchestra.run.smithi152.stdout:epoch 20 2024-04-02T04:56:49.277 INFO:teuthology.orchestra.run.smithi152.stdout:flags 32 joinable allow_snaps allow_multimds_snaps allow_standby_replay 2024-04-02T04:56:49.277 INFO:teuthology.orchestra.run.smithi152.stdout:created 2024-04-02T04:55:50.446442+0000 2024-04-02T04:56:49.277 INFO:teuthology.orchestra.run.smithi152.stdout:modified 2024-04-02T04:56:08.533665+0000 2024-04-02T04:56:49.277 INFO:teuthology.orchestra.run.smithi152.stdout:tableserver 0 2024-04-02T04:56:49.277 INFO:teuthology.orchestra.run.smithi152.stdout:root 0 2024-04-02T04:56:49.278 INFO:teuthology.orchestra.run.smithi152.stdout:session_timeout 60 2024-04-02T04:56:49.278 INFO:teuthology.orchestra.run.smithi152.stdout:session_autoclose 300 2024-04-02T04:56:49.278 INFO:teuthology.orchestra.run.smithi152.stdout:max_file_size 1099511627776 2024-04-02T04:56:49.278 INFO:teuthology.orchestra.run.smithi152.stdout:required_client_features {} 2024-04-02T04:56:49.278 INFO:teuthology.orchestra.run.smithi152.stdout:last_failure 0 2024-04-02T04:56:49.278 INFO:teuthology.orchestra.run.smithi152.stdout:last_failure_osd_epoch 46 2024-04-02T04:56:49.278 INFO:teuthology.orchestra.run.smithi152.stdout:compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-02T04:56:49.278 INFO:teuthology.orchestra.run.smithi152.stdout:max_mds 2 2024-04-02T04:56:49.278 INFO:teuthology.orchestra.run.smithi152.stdout:in 0,1 2024-04-02T04:56:49.278 INFO:teuthology.orchestra.run.smithi152.stdout:up {0=14456,1=14470} 2024-04-02T04:56:49.278 INFO:teuthology.orchestra.run.smithi152.stdout:failed 2024-04-02T04:56:49.278 INFO:teuthology.orchestra.run.smithi152.stdout:damaged 2024-04-02T04:56:49.278 INFO:teuthology.orchestra.run.smithi152.stdout:stopped 2024-04-02T04:56:49.278 INFO:teuthology.orchestra.run.smithi152.stdout:data_pools [3] 2024-04-02T04:56:49.279 INFO:teuthology.orchestra.run.smithi152.stdout:metadata_pool 2 2024-04-02T04:56:49.279 INFO:teuthology.orchestra.run.smithi152.stdout:inline_data enabled 2024-04-02T04:56:49.279 INFO:teuthology.orchestra.run.smithi152.stdout:balancer 2024-04-02T04:56:49.279 INFO:teuthology.orchestra.run.smithi152.stdout:standby_count_wanted 1 2024-04-02T04:56:49.279 INFO:teuthology.orchestra.run.smithi152.stdout:[mds.cephfs.smithi152.lluqrs{0:14456} state up:active seq 5 join_fscid=1 addr [v2:172.21.15.152:6826/4038990242,v1:172.21.15.152:6827/4038990242] compat {c=[1],r=[1],i=[7ff]}] 2024-04-02T04:56:49.279 INFO:teuthology.orchestra.run.smithi152.stdout:[mds.cephfs.smithi154.rbkmwp{0:24297} state up:standby-replay seq 2 join_fscid=1 addr [v2:172.21.15.154:6826/53707828,v1:172.21.15.154:6827/53707828] compat {c=[1],r=[1],i=[7ff]}] 2024-04-02T04:56:49.279 INFO:teuthology.orchestra.run.smithi152.stdout:[mds.cephfs.smithi152.ywghzy{1:14470} state up:active seq 6 join_fscid=1 addr [v2:172.21.15.152:6828/3554217123,v1:172.21.15.152:6829/3554217123] compat {c=[1],r=[1],i=[7ff]}] 2024-04-02T04:56:49.279 INFO:teuthology.orchestra.run.smithi152.stdout:[mds.cephfs.smithi154.llvaoh{1:24301} state up:standby-replay seq 1 join_fscid=1 addr [v2:172.21.15.154:6824/1569612467,v1:172.21.15.154:6825/1569612467] compat {c=[1],r=[1],i=[7ff]}] 2024-04-02T04:56:49.279 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:56:49.279 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:56:49.280 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.275+0000 7feadcd22700 1 -- 172.21.15.152:0/1589568443 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7feac40692a0 msgr2=0x7feac406b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:49.280 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.275+0000 7feadcd22700 1 --2- 172.21.15.152:0/1589568443 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7feac40692a0 0x7feac406b750 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 crypto rx=0x7feac0004f90 tx=0x7feac0009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:49.280 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.275+0000 7feadcd22700 1 -- 172.21.15.152:0/1589568443 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fead80fe4c0 msgr2=0x7fead810da30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:49.280 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.275+0000 7feadcd22700 1 --2- 172.21.15.152:0/1589568443 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fead80fe4c0 0x7fead810da30 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 crypto rx=0x7feac800c830 tx=0x7feac8004470 comp rx=0 tx=0).stop 2024-04-02T04:56:49.280 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.275+0000 7feadcd22700 1 -- 172.21.15.152:0/1589568443 shutdown_connections 2024-04-02T04:56:49.280 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.275+0000 7feadcd22700 1 --2- 172.21.15.152:0/1589568443 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7feac40692a0 0x7feac406b750 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:49.280 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.275+0000 7feadcd22700 1 --2- 172.21.15.152:0/1589568443 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fead80fe4c0 0x7fead810da30 unknown :-1 s=CLOSED pgs=285 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:49.280 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.275+0000 7feadcd22700 1 --2- 172.21.15.152:0/1589568443 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fead80fee10 0x7fead810df70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:49.280 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.275+0000 7feadcd22700 1 -- 172.21.15.152:0/1589568443 >> 172.21.15.152:0/1589568443 conn(0x7fead80f6ba0 msgr2=0x7fead80f82e0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:49.281 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.275+0000 7feadcd22700 1 -- 172.21.15.152:0/1589568443 shutdown_connections 2024-04-02T04:56:49.281 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.275+0000 7feadcd22700 1 -- 172.21.15.152:0/1589568443 wait complete. 2024-04-02T04:56:49.281 INFO:teuthology.orchestra.run.smithi152.stderr:dumped fsmap epoch 20 2024-04-02T04:56:49.390 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.384+0000 7f5206ed6700 1 -- 172.21.15.152:0/3982794619 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f52000fffc0 msgr2=0x7f52001023a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:49.390 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.384+0000 7f5206ed6700 1 --2- 172.21.15.152:0/3982794619 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f52000fffc0 0x7f52001023a0 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 crypto rx=0x7f51ec004940 tx=0x7f51ec009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:49.390 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.384+0000 7f5206ed6700 1 -- 172.21.15.152:0/3982794619 shutdown_connections 2024-04-02T04:56:49.390 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.384+0000 7f5206ed6700 1 --2- 172.21.15.152:0/3982794619 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f52000fffc0 0x7f52001023a0 unknown :-1 s=CLOSED pgs=286 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:49.390 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.384+0000 7f5206ed6700 1 --2- 172.21.15.152:0/3982794619 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5200066220 0x7f52000ffa80 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:49.390 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.384+0000 7f5206ed6700 1 -- 172.21.15.152:0/3982794619 >> 172.21.15.152:0/3982794619 conn(0x7f52000f7430 msgr2=0x7f52000f9860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:49.390 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.384+0000 7f5206ed6700 1 -- 172.21.15.152:0/3982794619 shutdown_connections 2024-04-02T04:56:49.390 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.384+0000 7f5206ed6700 1 -- 172.21.15.152:0/3982794619 wait complete. 2024-04-02T04:56:49.391 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.385+0000 7f5206ed6700 1 Processor -- start 2024-04-02T04:56:49.391 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.385+0000 7f5206ed6700 1 -- start start 2024-04-02T04:56:49.391 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.385+0000 7f5206ed6700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5200066220 0x7f520010da60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:49.391 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.385+0000 7f5206ed6700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f52000fffc0 0x7f520010dfa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:49.391 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.385+0000 7f5206ed6700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f520010e5a0 con 0x7f52000fffc0 2024-04-02T04:56:49.391 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.385+0000 7f5206ed6700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f520010edf0 con 0x7f5200066220 2024-04-02T04:56:49.391 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.385+0000 7f51fffff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f52000fffc0 0x7f520010dfa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:49.391 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.386+0000 7f51fffff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f52000fffc0 0x7f520010dfa0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:45260/0 (socket says 172.21.15.152:45260) 2024-04-02T04:56:49.391 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.386+0000 7f51fffff700 1 -- 172.21.15.152:0/2130432150 learned_addr learned my addr 172.21.15.152:0/2130432150 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:56:49.391 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.386+0000 7f5204c72700 1 --2- 172.21.15.152:0/2130432150 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5200066220 0x7f520010da60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:49.391 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.386+0000 7f51fffff700 1 -- 172.21.15.152:0/2130432150 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5200066220 msgr2=0x7f520010da60 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:49.391 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.386+0000 7f51fffff700 1 --2- 172.21.15.152:0/2130432150 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5200066220 0x7f520010da60 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:49.391 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.386+0000 7f51fffff700 1 -- 172.21.15.152:0/2130432150 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f51ec005040 con 0x7f52000fffc0 2024-04-02T04:56:49.392 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.386+0000 7f51fffff700 1 --2- 172.21.15.152:0/2130432150 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f52000fffc0 0x7f520010dfa0 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7f51ec00b970 tx=0x7f51ec009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:49.392 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.386+0000 7f51fd7fa700 1 -- 172.21.15.152:0/2130432150 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f51ec00e040 con 0x7f52000fffc0 2024-04-02T04:56:49.392 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.386+0000 7f5206ed6700 1 -- 172.21.15.152:0/2130432150 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f520010f070 con 0x7f52000fffc0 2024-04-02T04:56:49.392 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.386+0000 7f51fd7fa700 1 -- 172.21.15.152:0/2130432150 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f51ec00bbf0 con 0x7f52000fffc0 2024-04-02T04:56:49.392 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.386+0000 7f51fd7fa700 1 -- 172.21.15.152:0/2130432150 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f51ec01c3f0 con 0x7f52000fffc0 2024-04-02T04:56:49.392 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.386+0000 7f5206ed6700 1 -- 172.21.15.152:0/2130432150 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f520010f5b0 con 0x7f52000fffc0 2024-04-02T04:56:49.393 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.387+0000 7f5206ed6700 1 -- 172.21.15.152:0/2130432150 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f520005f5f0 con 0x7f52000fffc0 2024-04-02T04:56:49.393 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.388+0000 7f51fd7fa700 1 -- 172.21.15.152:0/2130432150 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f51ec00bd60 con 0x7f52000fffc0 2024-04-02T04:56:49.393 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.388+0000 7f51fd7fa700 1 --2- 172.21.15.152:0/2130432150 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f51f00692b0 0x7f51f006b760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:49.393 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.388+0000 7f51fd7fa700 1 -- 172.21.15.152:0/2130432150 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7f51ec088520 con 0x7f52000fffc0 2024-04-02T04:56:49.393 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.388+0000 7f5204c72700 1 --2- 172.21.15.152:0/2130432150 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f51f00692b0 0x7f51f006b760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:49.393 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.389+0000 7f5204c72700 1 --2- 172.21.15.152:0/2130432150 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f51f00692b0 0x7f51f006b760 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f5200065f60 tx=0x7f51f4009040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:49.397 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.393+0000 7f51fd7fa700 1 -- 172.21.15.152:0/2130432150 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f51ec058340 con 0x7f52000fffc0 2024-04-02T04:56:49.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:49 smithi154 ceph-mon[35729]: from='client.14548 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:56:49.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:49 smithi154 ceph-mon[35729]: pgmap v203: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 170 B/s wr, 4 op/s 2024-04-02T04:56:49.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:49 smithi154 ceph-mon[35729]: from='client.14552 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:56:49.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:49 smithi154 ceph-mon[35729]: from='client.14556 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:56:49.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:49 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/679680099' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T04:56:49.618 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.614+0000 7f5206ed6700 1 -- 172.21.15.152:0/2130432150 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}) v1 -- 0x7f520010f8e0 con 0x7f51f00692b0 2024-04-02T04:56:49.619 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.616+0000 7f51fd7fa700 1 -- 172.21.15.152:0/2130432150 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+358 (secure 0 0 0) 0x7f520010f8e0 con 0x7f51f00692b0 2024-04-02T04:56:49.620 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-04-02T04:56:49.620 INFO:teuthology.orchestra.run.smithi152.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca", 2024-04-02T04:56:49.620 INFO:teuthology.orchestra.run.smithi152.stdout: "in_progress": true, 2024-04-02T04:56:49.620 INFO:teuthology.orchestra.run.smithi152.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-02T04:56:49.620 INFO:teuthology.orchestra.run.smithi152.stdout: "services_complete": [], 2024-04-02T04:56:49.620 INFO:teuthology.orchestra.run.smithi152.stdout: "progress": "", 2024-04-02T04:56:49.620 INFO:teuthology.orchestra.run.smithi152.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:6cec793d9073f0594e2364b9b687ecaf22424cca image", 2024-04-02T04:56:49.620 INFO:teuthology.orchestra.run.smithi152.stdout: "is_paused": false 2024-04-02T04:56:49.620 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-04-02T04:56:49.622 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.618+0000 7f5206ed6700 1 -- 172.21.15.152:0/2130432150 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f51f00692b0 msgr2=0x7f51f006b760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:49.622 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.618+0000 7f5206ed6700 1 --2- 172.21.15.152:0/2130432150 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f51f00692b0 0x7f51f006b760 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 crypto rx=0x7f5200065f60 tx=0x7f51f4009040 comp rx=0 tx=0).stop 2024-04-02T04:56:49.622 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.618+0000 7f5206ed6700 1 -- 172.21.15.152:0/2130432150 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f52000fffc0 msgr2=0x7f520010dfa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:49.622 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.618+0000 7f5206ed6700 1 --2- 172.21.15.152:0/2130432150 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f52000fffc0 0x7f520010dfa0 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 crypto rx=0x7f51ec00b970 tx=0x7f51ec009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:49.623 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.619+0000 7f5206ed6700 1 -- 172.21.15.152:0/2130432150 shutdown_connections 2024-04-02T04:56:49.623 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.619+0000 7f5206ed6700 1 --2- 172.21.15.152:0/2130432150 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f51f00692b0 0x7f51f006b760 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:49.623 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.619+0000 7f5206ed6700 1 --2- 172.21.15.152:0/2130432150 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f5200066220 0x7f520010da60 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:49.623 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.619+0000 7f5206ed6700 1 --2- 172.21.15.152:0/2130432150 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f52000fffc0 0x7f520010dfa0 unknown :-1 s=CLOSED pgs=287 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:49.623 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.619+0000 7f5206ed6700 1 -- 172.21.15.152:0/2130432150 >> 172.21.15.152:0/2130432150 conn(0x7f52000f7430 msgr2=0x7f52000f7850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:49.623 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.619+0000 7f5206ed6700 1 -- 172.21.15.152:0/2130432150 shutdown_connections 2024-04-02T04:56:49.623 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.619+0000 7f5206ed6700 1 -- 172.21.15.152:0/2130432150 wait complete. 2024-04-02T04:56:49.732 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.728+0000 7fe03e96b700 1 -- 172.21.15.152:0/17147143 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe0380fff00 msgr2=0x7fe038102320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:49.732 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.728+0000 7fe03e96b700 1 --2- 172.21.15.152:0/17147143 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe0380fff00 0x7fe038102320 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 crypto rx=0x7fe02c004940 tx=0x7fe02c009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:49.732 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.728+0000 7fe03e96b700 1 -- 172.21.15.152:0/17147143 shutdown_connections 2024-04-02T04:56:49.732 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.728+0000 7fe03e96b700 1 --2- 172.21.15.152:0/17147143 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe0380fff00 0x7fe038102320 unknown :-1 s=CLOSED pgs=288 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:49.732 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.728+0000 7fe03e96b700 1 --2- 172.21.15.152:0/17147143 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe0380662e0 0x7fe0380ff9c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:49.732 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.728+0000 7fe03e96b700 1 -- 172.21.15.152:0/17147143 >> 172.21.15.152:0/17147143 conn(0x7fe0380f7430 msgr2=0x7fe0380f9860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:49.732 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.728+0000 7fe03e96b700 1 -- 172.21.15.152:0/17147143 shutdown_connections 2024-04-02T04:56:49.732 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.728+0000 7fe03e96b700 1 -- 172.21.15.152:0/17147143 wait complete. 2024-04-02T04:56:49.734 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.729+0000 7fe03e96b700 1 Processor -- start 2024-04-02T04:56:49.734 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.729+0000 7fe03e96b700 1 -- start start 2024-04-02T04:56:49.734 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.729+0000 7fe03e96b700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe0380662e0 0x7fe0380fd3f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:49.734 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.729+0000 7fe03e96b700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe0380fff00 0x7fe0380fb8c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:49.734 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.729+0000 7fe03e96b700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe0380fbe70 con 0x7fe0380662e0 2024-04-02T04:56:49.734 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.729+0000 7fe03e96b700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe0380fbfe0 con 0x7fe0380fff00 2024-04-02T04:56:49.734 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.729+0000 7fe037fff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe0380662e0 0x7fe0380fd3f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:49.734 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.729+0000 7fe037fff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe0380662e0 0x7fe0380fd3f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:45280/0 (socket says 172.21.15.152:45280) 2024-04-02T04:56:49.735 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.729+0000 7fe037fff700 1 -- 172.21.15.152:0/1622711639 learned_addr learned my addr 172.21.15.152:0/1622711639 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:56:49.735 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.729+0000 7fe0377fe700 1 --2- 172.21.15.152:0/1622711639 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe0380fff00 0x7fe0380fb8c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:49.735 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.730+0000 7fe037fff700 1 -- 172.21.15.152:0/1622711639 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe0380fff00 msgr2=0x7fe0380fb8c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:49.735 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.730+0000 7fe037fff700 1 --2- 172.21.15.152:0/1622711639 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe0380fff00 0x7fe0380fb8c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:49.735 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.730+0000 7fe037fff700 1 -- 172.21.15.152:0/1622711639 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe02c005040 con 0x7fe0380662e0 2024-04-02T04:56:49.735 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.730+0000 7fe037fff700 1 --2- 172.21.15.152:0/1622711639 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe0380662e0 0x7fe0380fd3f0 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7fe02800c800 tx=0x7fe028004470 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:49.735 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.730+0000 7fe034ff9700 1 -- 172.21.15.152:0/1622711639 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fe0280106f0 con 0x7fe0380662e0 2024-04-02T04:56:49.735 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.730+0000 7fe03e96b700 1 -- 172.21.15.152:0/1622711639 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe0380fc2c0 con 0x7fe0380662e0 2024-04-02T04:56:49.735 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.730+0000 7fe034ff9700 1 -- 172.21.15.152:0/1622711639 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fe028010d20 con 0x7fe0380662e0 2024-04-02T04:56:49.735 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.730+0000 7fe034ff9700 1 -- 172.21.15.152:0/1622711639 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fe0280113a0 con 0x7fe0380662e0 2024-04-02T04:56:49.735 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.730+0000 7fe03e96b700 1 -- 172.21.15.152:0/1622711639 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe038115a90 con 0x7fe0380662e0 2024-04-02T04:56:49.736 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.731+0000 7fe03e96b700 1 -- 172.21.15.152:0/1622711639 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe03805f5f0 con 0x7fe0380662e0 2024-04-02T04:56:49.736 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.732+0000 7fe034ff9700 1 -- 172.21.15.152:0/1622711639 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7fe028012400 con 0x7fe0380662e0 2024-04-02T04:56:49.736 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.732+0000 7fe034ff9700 1 --2- 172.21.15.152:0/1622711639 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fe0200692a0 0x7fe02006b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:56:49.736 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.732+0000 7fe034ff9700 1 -- 172.21.15.152:0/1622711639 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7fe028087bc0 con 0x7fe0380662e0 2024-04-02T04:56:49.736 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.732+0000 7fe0377fe700 1 --2- 172.21.15.152:0/1622711639 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fe0200692a0 0x7fe02006b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:56:49.737 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.733+0000 7fe0377fe700 1 --2- 172.21.15.152:0/1622711639 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fe0200692a0 0x7fe02006b750 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7fe02c00b970 tx=0x7fe02c009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:56:49.741 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:49.738+0000 7fe034ff9700 1 -- 172.21.15.152:0/1622711639 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fe028053f40 con 0x7fe0380662e0 2024-04-02T04:56:50.020 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:50.016+0000 7fe03e96b700 1 -- 172.21.15.152:0/1622711639 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "health", "detail": "detail"} v 0) v1 -- 0x7fe03804bed0 con 0x7fe0380662e0 2024-04-02T04:56:50.021 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:50.017+0000 7fe034ff9700 1 -- 172.21.15.152:0/1622711639 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "health", "detail": "detail"}]=0 v0) v1 ==== 74+0+201 (secure 0 0 0) 0x7fe028057560 con 0x7fe0380662e0 2024-04-02T04:56:50.021 INFO:teuthology.orchestra.run.smithi152.stdout:HEALTH_WARN 1 filesystem with deprecated feature inline_data 2024-04-02T04:56:50.021 INFO:teuthology.orchestra.run.smithi152.stdout:[WRN] FS_INLINE_DATA_DEPRECATED: 1 filesystem with deprecated feature inline_data 2024-04-02T04:56:50.021 INFO:teuthology.orchestra.run.smithi152.stdout: fs cephfs has deprecated feature inline_data enabled. 2024-04-02T04:56:50.023 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:50.019+0000 7fe03e96b700 1 -- 172.21.15.152:0/1622711639 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fe0200692a0 msgr2=0x7fe02006b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:50.023 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:50.019+0000 7fe03e96b700 1 --2- 172.21.15.152:0/1622711639 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fe0200692a0 0x7fe02006b750 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 crypto rx=0x7fe02c00b970 tx=0x7fe02c009e30 comp rx=0 tx=0).stop 2024-04-02T04:56:50.023 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:50.019+0000 7fe03e96b700 1 -- 172.21.15.152:0/1622711639 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe0380662e0 msgr2=0x7fe0380fd3f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:56:50.023 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:50.019+0000 7fe03e96b700 1 --2- 172.21.15.152:0/1622711639 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe0380662e0 0x7fe0380fd3f0 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 crypto rx=0x7fe02800c800 tx=0x7fe028004470 comp rx=0 tx=0).stop 2024-04-02T04:56:50.023 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:50.019+0000 7fe03e96b700 1 -- 172.21.15.152:0/1622711639 shutdown_connections 2024-04-02T04:56:50.023 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:50.019+0000 7fe03e96b700 1 --2- 172.21.15.152:0/1622711639 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fe0200692a0 0x7fe02006b750 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:50.023 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:50.019+0000 7fe03e96b700 1 --2- 172.21.15.152:0/1622711639 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fe0380662e0 0x7fe0380fd3f0 unknown :-1 s=CLOSED pgs=289 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:50.023 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:50.019+0000 7fe03e96b700 1 --2- 172.21.15.152:0/1622711639 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fe0380fff00 0x7fe0380fb8c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:56:50.024 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:50.019+0000 7fe03e96b700 1 -- 172.21.15.152:0/1622711639 >> 172.21.15.152:0/1622711639 conn(0x7fe0380f7430 msgr2=0x7fe0380f7850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:56:50.024 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:50.019+0000 7fe03e96b700 1 -- 172.21.15.152:0/1622711639 shutdown_connections 2024-04-02T04:56:50.024 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:56:50.019+0000 7fe03e96b700 1 -- 172.21.15.152:0/1622711639 wait complete. 2024-04-02T04:56:50.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:50 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/1589568443' entity='client.admin' cmd=[{"prefix": "fs dump"}]: dispatch 2024-04-02T04:56:50.233 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:50 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/1622711639' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T04:56:50.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:50 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/1589568443' entity='client.admin' cmd=[{"prefix": "fs dump"}]: dispatch 2024-04-02T04:56:50.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:50 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/1622711639' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T04:56:51.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:51 smithi154 ceph-mon[35729]: from='client.14568 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:56:51.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:51 smithi154 ceph-mon[35729]: pgmap v204: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-02T04:56:51.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:51 smithi152 ceph-mon[29073]: from='client.14568 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:56:51.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:51 smithi152 ceph-mon[29073]: pgmap v204: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-02T04:56:53.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:53 smithi154 ceph-mon[35729]: pgmap v205: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 3 op/s 2024-04-02T04:56:53.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:53 smithi152 ceph-mon[29073]: pgmap v205: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 3 op/s 2024-04-02T04:56:55.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:55 smithi154 ceph-mon[35729]: pgmap v206: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-02T04:56:55.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:55 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:56:55.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:55 smithi152 ceph-mon[29073]: pgmap v206: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-02T04:56:55.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:55 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:56:57.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:57 smithi154 ceph-mon[35729]: pgmap v207: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 3 op/s 2024-04-02T04:56:57.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:57 smithi152 ceph-mon[29073]: pgmap v207: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 3 op/s 2024-04-02T04:56:59.423 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:56:59 smithi152 ceph-mon[29073]: pgmap v208: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 3 op/s 2024-04-02T04:56:59.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:56:59 smithi154 ceph-mon[35729]: pgmap v208: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 3 op/s 2024-04-02T04:57:01.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:01 smithi154 ceph-mon[35729]: pgmap v209: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-02T04:57:01.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:01 smithi152 ceph-mon[29073]: pgmap v209: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-02T04:57:03.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:03 smithi154 ceph-mon[35729]: pgmap v210: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 3 op/s 2024-04-02T04:57:03.509 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:03 smithi152 ceph-mon[29073]: pgmap v210: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 3 op/s 2024-04-02T04:57:05.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:05 smithi154 ceph-mon[35729]: pgmap v211: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-02T04:57:05.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:05 smithi152 ceph-mon[29073]: pgmap v211: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-02T04:57:07.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:07 smithi154 ceph-mon[35729]: pgmap v212: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 3 op/s 2024-04-02T04:57:07.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:07 smithi152 ceph-mon[29073]: pgmap v212: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 3 op/s 2024-04-02T04:57:09.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:09 smithi154 ceph-mon[35729]: pgmap v213: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 3 op/s 2024-04-02T04:57:09.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:09 smithi152 ceph-mon[29073]: pgmap v213: 65 pgs: 65 active+clean; 586 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 3 op/s 2024-04-02T04:57:10.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:10 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:57:10.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:10 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:57:11.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:11 smithi154 ceph-mon[35729]: pgmap v214: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.8 KiB/s rd, 426 B/s wr, 4 op/s 2024-04-02T04:57:11.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:11 smithi152 ceph-mon[29073]: pgmap v214: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.8 KiB/s rd, 426 B/s wr, 4 op/s 2024-04-02T04:57:13.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:13 smithi154 ceph-mon[35729]: pgmap v215: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.0 KiB/s rd, 511 B/s wr, 4 op/s 2024-04-02T04:57:13.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:13 smithi152 ceph-mon[29073]: pgmap v215: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.0 KiB/s rd, 511 B/s wr, 4 op/s 2024-04-02T04:57:15.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:15 smithi154 ceph-mon[35729]: pgmap v216: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.8 KiB/s rd, 511 B/s wr, 5 op/s 2024-04-02T04:57:15.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:15 smithi152 ceph-mon[29073]: pgmap v216: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.8 KiB/s rd, 511 B/s wr, 5 op/s 2024-04-02T04:57:17.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:17 smithi154 ceph-mon[35729]: pgmap v217: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.0 KiB/s rd, 511 B/s wr, 3 op/s 2024-04-02T04:57:17.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:17 smithi152 ceph-mon[29073]: pgmap v217: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.0 KiB/s rd, 511 B/s wr, 3 op/s 2024-04-02T04:57:19.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:19 smithi154 ceph-mon[35729]: pgmap v218: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.0 KiB/s rd, 597 B/s wr, 4 op/s 2024-04-02T04:57:19.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:19 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:57:19.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:19 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:57:19.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:19 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:57:19.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:19 smithi152 ceph-mon[29073]: pgmap v218: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.0 KiB/s rd, 597 B/s wr, 4 op/s 2024-04-02T04:57:19.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:19 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:57:19.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:19 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:57:19.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:19 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:57:20.133 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.128+0000 7fd3ac23a700 1 -- 172.21.15.152:0/3213624517 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd3a40fcf80 msgr2=0x7fd3a40fd3f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:20.133 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.128+0000 7fd3ac23a700 1 --2- 172.21.15.152:0/3213624517 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd3a40fcf80 0x7fd3a40fd3f0 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 crypto rx=0x7fd394004940 tx=0x7fd394009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:20.133 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.128+0000 7fd3ac23a700 1 -- 172.21.15.152:0/3213624517 shutdown_connections 2024-04-02T04:57:20.133 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.128+0000 7fd3ac23a700 1 --2- 172.21.15.152:0/3213624517 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd3a40fcf80 0x7fd3a40fd3f0 unknown :-1 s=CLOSED pgs=290 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:20.133 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.128+0000 7fd3ac23a700 1 --2- 172.21.15.152:0/3213624517 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd3a40fbcd0 0x7fd3a40fc0e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:20.133 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.128+0000 7fd3ac23a700 1 -- 172.21.15.152:0/3213624517 >> 172.21.15.152:0/3213624517 conn(0x7fd3a40f7430 msgr2=0x7fd3a40f9860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:20.133 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.128+0000 7fd3ac23a700 1 -- 172.21.15.152:0/3213624517 shutdown_connections 2024-04-02T04:57:20.133 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.128+0000 7fd3ac23a700 1 -- 172.21.15.152:0/3213624517 wait complete. 2024-04-02T04:57:20.134 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.129+0000 7fd3ac23a700 1 Processor -- start 2024-04-02T04:57:20.134 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.129+0000 7fd3ac23a700 1 -- start start 2024-04-02T04:57:20.134 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.129+0000 7fd3ac23a700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd3a40fbcd0 0x7fd3a4105fd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:20.134 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.129+0000 7fd3ac23a700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd3a40fcf80 0x7fd3a4106510 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:20.134 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.129+0000 7fd3ac23a700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd3a4106b60 con 0x7fd3a40fcf80 2024-04-02T04:57:20.134 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.129+0000 7fd3ac23a700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd3a41073b0 con 0x7fd3a40fbcd0 2024-04-02T04:57:20.135 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.129+0000 7fd3a97d5700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd3a40fcf80 0x7fd3a4106510 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:20.135 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.129+0000 7fd3a97d5700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd3a40fcf80 0x7fd3a4106510 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:34924/0 (socket says 172.21.15.152:34924) 2024-04-02T04:57:20.135 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.129+0000 7fd3a97d5700 1 -- 172.21.15.152:0/579787285 learned_addr learned my addr 172.21.15.152:0/579787285 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:57:20.135 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.130+0000 7fd3a9fd6700 1 --2- 172.21.15.152:0/579787285 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd3a40fbcd0 0x7fd3a4105fd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:20.136 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.130+0000 7fd3a97d5700 1 -- 172.21.15.152:0/579787285 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd3a40fbcd0 msgr2=0x7fd3a4105fd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:20.136 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.130+0000 7fd3a97d5700 1 --2- 172.21.15.152:0/579787285 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd3a40fbcd0 0x7fd3a4105fd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:20.136 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.130+0000 7fd3a97d5700 1 -- 172.21.15.152:0/579787285 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd394005040 con 0x7fd3a40fcf80 2024-04-02T04:57:20.136 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.130+0000 7fd3a97d5700 1 --2- 172.21.15.152:0/579787285 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd3a40fcf80 0x7fd3a4106510 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7fd394013870 tx=0x7fd394009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:20.136 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.130+0000 7fd39a7fc700 1 -- 172.21.15.152:0/579787285 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fd394021070 con 0x7fd3a40fcf80 2024-04-02T04:57:20.136 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.130+0000 7fd3ac23a700 1 -- 172.21.15.152:0/579787285 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd3a4107630 con 0x7fd3a40fcf80 2024-04-02T04:57:20.136 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.130+0000 7fd39a7fc700 1 -- 172.21.15.152:0/579787285 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fd394009430 con 0x7fd3a40fcf80 2024-04-02T04:57:20.136 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.130+0000 7fd39a7fc700 1 -- 172.21.15.152:0/579787285 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fd39400e070 con 0x7fd3a40fcf80 2024-04-02T04:57:20.136 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.130+0000 7fd3ac23a700 1 -- 172.21.15.152:0/579787285 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd3a4107b70 con 0x7fd3a40fcf80 2024-04-02T04:57:20.136 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.132+0000 7fd3ac23a700 1 -- 172.21.15.152:0/579787285 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd3a4101910 con 0x7fd3a40fcf80 2024-04-02T04:57:20.137 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.132+0000 7fd39a7fc700 1 -- 172.21.15.152:0/579787285 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7fd39401c750 con 0x7fd3a40fcf80 2024-04-02T04:57:20.137 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.133+0000 7fd39a7fc700 1 --2- 172.21.15.152:0/579787285 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd390069230 0x7fd39006b6e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:20.137 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.133+0000 7fd39a7fc700 1 -- 172.21.15.152:0/579787285 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7fd3940884d0 con 0x7fd3a40fcf80 2024-04-02T04:57:20.137 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.133+0000 7fd3a9fd6700 1 --2- 172.21.15.152:0/579787285 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd390069230 0x7fd39006b6e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:20.137 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.134+0000 7fd3a9fd6700 1 --2- 172.21.15.152:0/579787285 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd390069230 0x7fd39006b6e0 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7fd3a4105d60 tx=0x7fd3a000b040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:20.146 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.142+0000 7fd39a7fc700 1 -- 172.21.15.152:0/579787285 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fd3940547b0 con 0x7fd3a40fcf80 2024-04-02T04:57:20.396 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.392+0000 7fd3ac23a700 1 -- 172.21.15.152:0/579787285 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}) v1 -- 0x7fd3a4104850 con 0x7fd390069230 2024-04-02T04:57:20.396 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:20 smithi152 ceph-mon[29073]: Upgrade: Target is version 19.0.0-1979-g6cec793d (unknown) 2024-04-02T04:57:20.396 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:20 smithi152 ceph-mon[29073]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T04:57:20.396 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:20 smithi152 ceph-mon[29073]: Upgrade: Need to upgrade myself (mgr.smithi152.jivdcl) 2024-04-02T04:57:20.396 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:20 smithi152 ceph-mon[29073]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae on smithi154 2024-04-02T04:57:20.397 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.393+0000 7fd39a7fc700 1 -- 172.21.15.152:0/579787285 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+447 (secure 0 0 0) 0x7fd3a4104850 con 0x7fd390069230 2024-04-02T04:57:20.399 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.396+0000 7fd3ac23a700 1 -- 172.21.15.152:0/579787285 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd390069230 msgr2=0x7fd39006b6e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:20.399 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.396+0000 7fd3ac23a700 1 --2- 172.21.15.152:0/579787285 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd390069230 0x7fd39006b6e0 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 crypto rx=0x7fd3a4105d60 tx=0x7fd3a000b040 comp rx=0 tx=0).stop 2024-04-02T04:57:20.400 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.396+0000 7fd3ac23a700 1 -- 172.21.15.152:0/579787285 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd3a40fcf80 msgr2=0x7fd3a4106510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:20.400 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.396+0000 7fd3ac23a700 1 --2- 172.21.15.152:0/579787285 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd3a40fcf80 0x7fd3a4106510 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7fd394013870 tx=0x7fd394009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:20.400 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.396+0000 7fd3ac23a700 1 -- 172.21.15.152:0/579787285 shutdown_connections 2024-04-02T04:57:20.400 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.396+0000 7fd3ac23a700 1 --2- 172.21.15.152:0/579787285 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fd390069230 0x7fd39006b6e0 unknown :-1 s=CLOSED pgs=123 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:20.400 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.396+0000 7fd3ac23a700 1 --2- 172.21.15.152:0/579787285 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fd3a40fbcd0 0x7fd3a4105fd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:20.400 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.396+0000 7fd3ac23a700 1 --2- 172.21.15.152:0/579787285 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fd3a40fcf80 0x7fd3a4106510 secure :-1 s=CLOSED pgs=291 cs=0 l=1 rev1=1 crypto rx=0x7fd394013870 tx=0x7fd394009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:20.400 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.396+0000 7fd3ac23a700 1 -- 172.21.15.152:0/579787285 >> 172.21.15.152:0/579787285 conn(0x7fd3a40f7430 msgr2=0x7fd3a4065b20 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:20.400 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.396+0000 7fd3ac23a700 1 -- 172.21.15.152:0/579787285 shutdown_connections 2024-04-02T04:57:20.400 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.396+0000 7fd3ac23a700 1 -- 172.21.15.152:0/579787285 wait complete. 2024-04-02T04:57:20.411 INFO:teuthology.orchestra.run.smithi152.stdout:true 2024-04-02T04:57:20.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:20 smithi154 ceph-mon[35729]: Upgrade: Target is version 19.0.0-1979-g6cec793d (unknown) 2024-04-02T04:57:20.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:20 smithi154 ceph-mon[35729]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae, digests ['quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae'] 2024-04-02T04:57:20.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:20 smithi154 ceph-mon[35729]: Upgrade: Need to upgrade myself (mgr.smithi152.jivdcl) 2024-04-02T04:57:20.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:20 smithi154 ceph-mon[35729]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae on smithi154 2024-04-02T04:57:20.507 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.502+0000 7f2c617a3700 1 -- 172.21.15.152:0/4106395588 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2c5c0fbcd0 msgr2=0x7f2c5c0fc0e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:20.507 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.502+0000 7f2c617a3700 1 --2- 172.21.15.152:0/4106395588 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2c5c0fbcd0 0x7f2c5c0fc0e0 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 crypto rx=0x7f2c4c004940 tx=0x7f2c4c009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:20.507 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.502+0000 7f2c617a3700 1 -- 172.21.15.152:0/4106395588 shutdown_connections 2024-04-02T04:57:20.507 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.502+0000 7f2c617a3700 1 --2- 172.21.15.152:0/4106395588 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2c5c0fcf80 0x7f2c5c0fd3f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:20.507 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.502+0000 7f2c617a3700 1 --2- 172.21.15.152:0/4106395588 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2c5c0fbcd0 0x7f2c5c0fc0e0 unknown :-1 s=CLOSED pgs=292 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:20.507 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.502+0000 7f2c617a3700 1 -- 172.21.15.152:0/4106395588 >> 172.21.15.152:0/4106395588 conn(0x7f2c5c0f7430 msgr2=0x7f2c5c0f9860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:20.507 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.503+0000 7f2c617a3700 1 -- 172.21.15.152:0/4106395588 shutdown_connections 2024-04-02T04:57:20.507 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.503+0000 7f2c617a3700 1 -- 172.21.15.152:0/4106395588 wait complete. 2024-04-02T04:57:20.509 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.504+0000 7f2c617a3700 1 Processor -- start 2024-04-02T04:57:20.509 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.504+0000 7f2c617a3700 1 -- start start 2024-04-02T04:57:20.509 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.504+0000 7f2c617a3700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2c5c0fcf80 0x7f2c5c10ca70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:20.509 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.504+0000 7f2c617a3700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2c5c10cfb0 0x7f2c5c10de00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:20.510 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.504+0000 7f2c617a3700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2c5c10d470 con 0x7f2c5c10cfb0 2024-04-02T04:57:20.510 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.504+0000 7f2c617a3700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2c5c10d5b0 con 0x7f2c5c0fcf80 2024-04-02T04:57:20.510 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.504+0000 7f2c5a7fc700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2c5c10cfb0 0x7f2c5c10de00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:20.510 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.504+0000 7f2c5affd700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2c5c0fcf80 0x7f2c5c10ca70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:20.510 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.504+0000 7f2c5a7fc700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2c5c10cfb0 0x7f2c5c10de00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:42252/0 (socket says 172.21.15.152:42252) 2024-04-02T04:57:20.510 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.504+0000 7f2c5a7fc700 1 -- 172.21.15.152:0/2665709604 learned_addr learned my addr 172.21.15.152:0/2665709604 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:57:20.510 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.504+0000 7f2c5a7fc700 1 -- 172.21.15.152:0/2665709604 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2c5c0fcf80 msgr2=0x7f2c5c10ca70 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:20.510 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.504+0000 7f2c5a7fc700 1 --2- 172.21.15.152:0/2665709604 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2c5c0fcf80 0x7f2c5c10ca70 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:20.510 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.505+0000 7f2c5a7fc700 1 -- 172.21.15.152:0/2665709604 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2c4c005040 con 0x7f2c5c10cfb0 2024-04-02T04:57:20.510 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.505+0000 7f2c5affd700 1 --2- 172.21.15.152:0/2665709604 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2c5c0fcf80 0x7f2c5c10ca70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-02T04:57:20.510 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.505+0000 7f2c5a7fc700 1 --2- 172.21.15.152:0/2665709604 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2c5c10cfb0 0x7f2c5c10de00 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7f2c5000eb40 tx=0x7f2c5000b040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:20.510 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.505+0000 7f2c437fe700 1 -- 172.21.15.152:0/2665709604 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f2c50007cc0 con 0x7f2c5c10cfb0 2024-04-02T04:57:20.510 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.505+0000 7f2c617a3700 1 -- 172.21.15.152:0/2665709604 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2c5c10e400 con 0x7f2c5c10cfb0 2024-04-02T04:57:20.511 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.505+0000 7f2c437fe700 1 -- 172.21.15.152:0/2665709604 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f2c5000f040 con 0x7f2c5c10cfb0 2024-04-02T04:57:20.511 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.505+0000 7f2c437fe700 1 -- 172.21.15.152:0/2665709604 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f2c5000ca10 con 0x7f2c5c10cfb0 2024-04-02T04:57:20.511 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.505+0000 7f2c617a3700 1 -- 172.21.15.152:0/2665709604 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2c5c10e910 con 0x7f2c5c10cfb0 2024-04-02T04:57:20.511 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.506+0000 7f2c617a3700 1 -- 172.21.15.152:0/2665709604 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2c5c05f5f0 con 0x7f2c5c10cfb0 2024-04-02T04:57:20.512 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.508+0000 7f2c437fe700 1 -- 172.21.15.152:0/2665709604 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f2c5000cc80 con 0x7f2c5c10cfb0 2024-04-02T04:57:20.512 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.508+0000 7f2c437fe700 1 --2- 172.21.15.152:0/2665709604 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2c440692b0 0x7f2c4406b760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:20.512 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.509+0000 7f2c437fe700 1 -- 172.21.15.152:0/2665709604 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7f2c50087010 con 0x7f2c5c10cfb0 2024-04-02T04:57:20.513 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.509+0000 7f2c5affd700 1 --2- 172.21.15.152:0/2665709604 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2c440692b0 0x7f2c4406b760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:20.513 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.509+0000 7f2c5affd700 1 --2- 172.21.15.152:0/2665709604 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2c440692b0 0x7f2c4406b760 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f2c4c000c00 tx=0x7f2c4c009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:20.516 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.513+0000 7f2c437fe700 1 -- 172.21.15.152:0/2665709604 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f2c50056e30 con 0x7f2c5c10cfb0 2024-04-02T04:57:20.726 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.722+0000 7f2c617a3700 1 -- 172.21.15.152:0/2665709604 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}) v1 -- 0x7f2c5c101420 con 0x7f2c440692b0 2024-04-02T04:57:20.727 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.723+0000 7f2c437fe700 1 -- 172.21.15.152:0/2665709604 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+447 (secure 0 0 0) 0x7f2c5c101420 con 0x7f2c440692b0 2024-04-02T04:57:20.730 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.726+0000 7f2c617a3700 1 -- 172.21.15.152:0/2665709604 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2c440692b0 msgr2=0x7f2c4406b760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:20.730 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.726+0000 7f2c617a3700 1 --2- 172.21.15.152:0/2665709604 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2c440692b0 0x7f2c4406b760 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 crypto rx=0x7f2c4c000c00 tx=0x7f2c4c009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:20.730 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.726+0000 7f2c617a3700 1 -- 172.21.15.152:0/2665709604 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2c5c10cfb0 msgr2=0x7f2c5c10de00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:20.730 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.726+0000 7f2c617a3700 1 --2- 172.21.15.152:0/2665709604 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2c5c10cfb0 0x7f2c5c10de00 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 crypto rx=0x7f2c5000eb40 tx=0x7f2c5000b040 comp rx=0 tx=0).stop 2024-04-02T04:57:20.731 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.726+0000 7f2c617a3700 1 -- 172.21.15.152:0/2665709604 shutdown_connections 2024-04-02T04:57:20.731 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.726+0000 7f2c617a3700 1 --2- 172.21.15.152:0/2665709604 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f2c440692b0 0x7f2c4406b760 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:20.731 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.726+0000 7f2c617a3700 1 --2- 172.21.15.152:0/2665709604 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f2c5c0fcf80 0x7f2c5c10ca70 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:20.731 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.726+0000 7f2c617a3700 1 --2- 172.21.15.152:0/2665709604 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f2c5c10cfb0 0x7f2c5c10de00 unknown :-1 s=CLOSED pgs=293 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:20.731 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.726+0000 7f2c617a3700 1 -- 172.21.15.152:0/2665709604 >> 172.21.15.152:0/2665709604 conn(0x7f2c5c0f7430 msgr2=0x7f2c5c0f8bf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:20.731 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.726+0000 7f2c617a3700 1 -- 172.21.15.152:0/2665709604 shutdown_connections 2024-04-02T04:57:20.731 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.726+0000 7f2c617a3700 1 -- 172.21.15.152:0/2665709604 wait complete. 2024-04-02T04:57:20.839 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.835+0000 7f56fd1dd700 1 -- 172.21.15.152:0/2734492888 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f56f80fcf80 msgr2=0x7f56f80fd3f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:20.839 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.835+0000 7f56fd1dd700 1 --2- 172.21.15.152:0/2734492888 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f56f80fcf80 0x7f56f80fd3f0 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 crypto rx=0x7f56ec004940 tx=0x7f56ec009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:20.839 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.835+0000 7f56fd1dd700 1 -- 172.21.15.152:0/2734492888 shutdown_connections 2024-04-02T04:57:20.839 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.835+0000 7f56fd1dd700 1 --2- 172.21.15.152:0/2734492888 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f56f80fcf80 0x7f56f80fd3f0 unknown :-1 s=CLOSED pgs=294 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:20.840 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.835+0000 7f56fd1dd700 1 --2- 172.21.15.152:0/2734492888 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f56f80fbcd0 0x7f56f80fc0e0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:20.840 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.835+0000 7f56fd1dd700 1 -- 172.21.15.152:0/2734492888 >> 172.21.15.152:0/2734492888 conn(0x7f56f80f7430 msgr2=0x7f56f80f9860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:20.840 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.835+0000 7f56fd1dd700 1 -- 172.21.15.152:0/2734492888 shutdown_connections 2024-04-02T04:57:20.840 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.835+0000 7f56fd1dd700 1 -- 172.21.15.152:0/2734492888 wait complete. 2024-04-02T04:57:20.841 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.836+0000 7f56fd1dd700 1 Processor -- start 2024-04-02T04:57:20.841 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.836+0000 7f56fd1dd700 1 -- start start 2024-04-02T04:57:20.841 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.836+0000 7f56fd1dd700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f56f80fbcd0 0x7f56f80ffdf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:20.841 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.836+0000 7f56fd1dd700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f56f80fcf80 0x7f56f80fe2c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:20.841 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.836+0000 7f56fd1dd700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f56f80fe870 con 0x7f56f80fcf80 2024-04-02T04:57:20.841 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.836+0000 7f56fd1dd700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f56f80fe9e0 con 0x7f56f80fbcd0 2024-04-02T04:57:20.842 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.837+0000 7f56f659c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f56f80fcf80 0x7f56f80fe2c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:20.842 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.837+0000 7f56f659c700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f56f80fcf80 0x7f56f80fe2c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:42270/0 (socket says 172.21.15.152:42270) 2024-04-02T04:57:20.842 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.837+0000 7f56f659c700 1 -- 172.21.15.152:0/3028920347 learned_addr learned my addr 172.21.15.152:0/3028920347 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:57:20.842 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.837+0000 7f56f6d9d700 1 --2- 172.21.15.152:0/3028920347 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f56f80fbcd0 0x7f56f80ffdf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:20.842 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.837+0000 7f56f659c700 1 -- 172.21.15.152:0/3028920347 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f56f80fbcd0 msgr2=0x7f56f80ffdf0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:20.842 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.837+0000 7f56f659c700 1 --2- 172.21.15.152:0/3028920347 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f56f80fbcd0 0x7f56f80ffdf0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:20.842 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.837+0000 7f56f659c700 1 -- 172.21.15.152:0/3028920347 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f56ec005040 con 0x7f56f80fcf80 2024-04-02T04:57:20.842 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.837+0000 7f56f659c700 1 --2- 172.21.15.152:0/3028920347 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f56f80fcf80 0x7f56f80fe2c0 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7f56ec013870 tx=0x7f56ec009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:20.842 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.838+0000 7f56df7fe700 1 -- 172.21.15.152:0/3028920347 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f56ec00e070 con 0x7f56f80fcf80 2024-04-02T04:57:20.843 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.838+0000 7f56df7fe700 1 -- 172.21.15.152:0/3028920347 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f56ec013de0 con 0x7f56f80fcf80 2024-04-02T04:57:20.843 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.838+0000 7f56fd1dd700 1 -- 172.21.15.152:0/3028920347 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f56f80fec60 con 0x7f56f80fcf80 2024-04-02T04:57:20.843 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.838+0000 7f56df7fe700 1 -- 172.21.15.152:0/3028920347 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f56ec01cea0 con 0x7f56f80fcf80 2024-04-02T04:57:20.843 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.838+0000 7f56fd1dd700 1 -- 172.21.15.152:0/3028920347 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f56f80ff080 con 0x7f56f80fcf80 2024-04-02T04:57:20.843 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.839+0000 7f56fd1dd700 1 -- 172.21.15.152:0/3028920347 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f56f8101b10 con 0x7f56f80fcf80 2024-04-02T04:57:20.844 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.840+0000 7f56df7fe700 1 -- 172.21.15.152:0/3028920347 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f56ec01c690 con 0x7f56f80fcf80 2024-04-02T04:57:20.844 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.840+0000 7f56df7fe700 1 --2- 172.21.15.152:0/3028920347 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f56e0069380 0x7f56e006b830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:20.844 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.840+0000 7f56df7fe700 1 -- 172.21.15.152:0/3028920347 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7f56ec0888d0 con 0x7f56f80fcf80 2024-04-02T04:57:20.844 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.840+0000 7f56f6d9d700 1 --2- 172.21.15.152:0/3028920347 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f56e0069380 0x7f56e006b830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:20.845 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.841+0000 7f56f6d9d700 1 --2- 172.21.15.152:0/3028920347 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f56e0069380 0x7f56e006b830 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f56f80ffb80 tx=0x7f56e800b040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:20.852 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:20.848+0000 7f56df7fe700 1 -- 172.21.15.152:0/3028920347 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f56ec054b90 con 0x7f56f80fcf80 2024-04-02T04:57:21.059 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.055+0000 7f56fd1dd700 1 -- 172.21.15.152:0/3028920347 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "orch ps", "target": ["mon-mgr", ""]}) v1 -- 0x7f56f805e400 con 0x7f56e0069380 2024-04-02T04:57:21.076 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.072+0000 7f56df7fe700 1 -- 172.21.15.152:0/3028920347 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+3366 (secure 0 0 0) 0x7f56f805e400 con 0x7f56e0069380 2024-04-02T04:57:21.076 INFO:teuthology.orchestra.run.smithi152.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T04:57:21.076 INFO:teuthology.orchestra.run.smithi152.stdout:alertmanager.smithi152 smithi152 *:9093,9094 running (5m) 61s ago 6m 25.2M - 0.25.0 c8568f914cd2 1f52e384342b 2024-04-02T04:57:21.076 INFO:teuthology.orchestra.run.smithi152.stdout:crash.smithi152 smithi152 running (7m) 61s ago 7m 6665k - 17.2.7-803-gc924f991 4948510fac0f 12f8a8489975 2024-04-02T04:57:21.076 INFO:teuthology.orchestra.run.smithi152.stdout:crash.smithi154 smithi154 running (6m) 67s ago 6m 6665k - 17.2.7-803-gc924f991 4948510fac0f 7fe215d49e0f 2024-04-02T04:57:21.076 INFO:teuthology.orchestra.run.smithi152.stdout:grafana.smithi152 smithi152 *:3000 running (5m) 61s ago 6m 85.7M - 9.4.7 954c08fa6188 eb6036b14f99 2024-04-02T04:57:21.076 INFO:teuthology.orchestra.run.smithi152.stdout:mds.cephfs.smithi152.lluqrs smithi152 running (87s) 61s ago 86s 15.0M - 17.2.7-803-gc924f991 4948510fac0f bada96a57e65 2024-04-02T04:57:21.076 INFO:teuthology.orchestra.run.smithi152.stdout:mds.cephfs.smithi152.ywghzy smithi152 running (81s) 61s ago 81s 15.9M - 17.2.7-803-gc924f991 4948510fac0f 267b6d51cad3 2024-04-02T04:57:21.077 INFO:teuthology.orchestra.run.smithi152.stdout:mds.cephfs.smithi154.llvaoh smithi154 running (84s) 67s ago 84s 11.3M - 17.2.7-803-gc924f991 4948510fac0f 82f2bed61253 2024-04-02T04:57:21.077 INFO:teuthology.orchestra.run.smithi152.stdout:mds.cephfs.smithi154.rbkmwp smithi154 running (78s) 67s ago 78s 14.7M - 17.2.7-803-gc924f991 4948510fac0f eba3a7f41a09 2024-04-02T04:57:21.077 INFO:teuthology.orchestra.run.smithi152.stdout:mgr.smithi152.jivdcl smithi152 *:9283 running (8m) 61s ago 8m 481M - 17.2.7-803-gc924f991 4948510fac0f 2fd5fbea3f34 2024-04-02T04:57:21.077 INFO:teuthology.orchestra.run.smithi152.stdout:mgr.smithi154.pplnxz smithi154 *:8443,9283 running (5m) 67s ago 5m 432M - 17.2.7-803-gc924f991 4948510fac0f 0a1df688b4ac 2024-04-02T04:57:21.077 INFO:teuthology.orchestra.run.smithi152.stdout:mon.smithi152 smithi152 running (8m) 61s ago 8m 46.3M 2048M 17.2.7-803-gc924f991 4948510fac0f 399b9cc778f0 2024-04-02T04:57:21.077 INFO:teuthology.orchestra.run.smithi152.stdout:mon.smithi154 smithi154 running (5m) 67s ago 5m 36.0M 2048M 17.2.7-803-gc924f991 4948510fac0f 975aefa2215f 2024-04-02T04:57:21.077 INFO:teuthology.orchestra.run.smithi152.stdout:node-exporter.smithi152 smithi152 *:9100 running (6m) 61s ago 6m 16.5M - 1.5.0 0da6a335fe13 8168d9a53e30 2024-04-02T04:57:21.077 INFO:teuthology.orchestra.run.smithi152.stdout:node-exporter.smithi154 smithi154 *:9100 running (5m) 67s ago 5m 15.7M - 1.5.0 0da6a335fe13 8ee2f7610052 2024-04-02T04:57:21.077 INFO:teuthology.orchestra.run.smithi152.stdout:osd.0 smithi152 running (5m) 61s ago 5m 45.0M 4288M 17.2.7-803-gc924f991 4948510fac0f 9935e01f3199 2024-04-02T04:57:21.077 INFO:teuthology.orchestra.run.smithi152.stdout:osd.1 smithi152 running (4m) 61s ago 4m 46.5M 4288M 17.2.7-803-gc924f991 4948510fac0f e5c49d238944 2024-04-02T04:57:21.077 INFO:teuthology.orchestra.run.smithi152.stdout:osd.2 smithi152 running (4m) 61s ago 4m 47.1M 4288M 17.2.7-803-gc924f991 4948510fac0f c53a93ae7737 2024-04-02T04:57:21.077 INFO:teuthology.orchestra.run.smithi152.stdout:osd.3 smithi154 running (3m) 67s ago 3m 44.8M 5312M 17.2.7-803-gc924f991 4948510fac0f 959ebaab3713 2024-04-02T04:57:21.077 INFO:teuthology.orchestra.run.smithi152.stdout:osd.4 smithi154 running (3m) 67s ago 3m 41.7M 5312M 17.2.7-803-gc924f991 4948510fac0f 4856c813e7fd 2024-04-02T04:57:21.077 INFO:teuthology.orchestra.run.smithi152.stdout:osd.5 smithi154 running (2m) 67s ago 2m 42.0M 5312M 17.2.7-803-gc924f991 4948510fac0f fbc6bb91d1a7 2024-04-02T04:57:21.077 INFO:teuthology.orchestra.run.smithi152.stdout:prometheus.smithi152 smithi152 *:9095 running (5m) 61s ago 6m 53.8M - 2.43.0 a07b618ecd1d 874955f0a102 2024-04-02T04:57:21.078 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.075+0000 7f56fd1dd700 1 -- 172.21.15.152:0/3028920347 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f56e0069380 msgr2=0x7f56e006b830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:21.078 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.075+0000 7f56fd1dd700 1 --2- 172.21.15.152:0/3028920347 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f56e0069380 0x7f56e006b830 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 crypto rx=0x7f56f80ffb80 tx=0x7f56e800b040 comp rx=0 tx=0).stop 2024-04-02T04:57:21.079 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.075+0000 7f56fd1dd700 1 -- 172.21.15.152:0/3028920347 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f56f80fcf80 msgr2=0x7f56f80fe2c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:21.079 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.075+0000 7f56fd1dd700 1 --2- 172.21.15.152:0/3028920347 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f56f80fcf80 0x7f56f80fe2c0 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 crypto rx=0x7f56ec013870 tx=0x7f56ec009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:21.079 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.075+0000 7f56fd1dd700 1 -- 172.21.15.152:0/3028920347 shutdown_connections 2024-04-02T04:57:21.079 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.075+0000 7f56fd1dd700 1 --2- 172.21.15.152:0/3028920347 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f56e0069380 0x7f56e006b830 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:21.079 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.075+0000 7f56fd1dd700 1 --2- 172.21.15.152:0/3028920347 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f56f80fbcd0 0x7f56f80ffdf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:21.079 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.075+0000 7f56fd1dd700 1 --2- 172.21.15.152:0/3028920347 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f56f80fcf80 0x7f56f80fe2c0 unknown :-1 s=CLOSED pgs=295 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:21.079 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.075+0000 7f56fd1dd700 1 -- 172.21.15.152:0/3028920347 >> 172.21.15.152:0/3028920347 conn(0x7f56f80f7430 msgr2=0x7f56f8065ae0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:21.079 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.075+0000 7f56fd1dd700 1 -- 172.21.15.152:0/3028920347 shutdown_connections 2024-04-02T04:57:21.079 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.075+0000 7f56fd1dd700 1 -- 172.21.15.152:0/3028920347 wait complete. 2024-04-02T04:57:21.189 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.184+0000 7f6979d75700 1 -- 172.21.15.152:0/3338957198 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f69740fcf90 msgr2=0x7f69740fd400 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:21.189 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.184+0000 7f6979d75700 1 --2- 172.21.15.152:0/3338957198 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f69740fcf90 0x7f69740fd400 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 crypto rx=0x7f6968004940 tx=0x7f6968009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:21.189 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.184+0000 7f6979d75700 1 -- 172.21.15.152:0/3338957198 shutdown_connections 2024-04-02T04:57:21.189 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.184+0000 7f6979d75700 1 --2- 172.21.15.152:0/3338957198 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f69740fcf90 0x7f69740fd400 unknown :-1 s=CLOSED pgs=296 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:21.189 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.184+0000 7f6979d75700 1 --2- 172.21.15.152:0/3338957198 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f69740fbce0 0x7f69740fc0f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:21.189 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.184+0000 7f6979d75700 1 -- 172.21.15.152:0/3338957198 >> 172.21.15.152:0/3338957198 conn(0x7f69740f7480 msgr2=0x7f69740f9890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:21.189 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.185+0000 7f6979d75700 1 -- 172.21.15.152:0/3338957198 shutdown_connections 2024-04-02T04:57:21.189 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.185+0000 7f6979d75700 1 -- 172.21.15.152:0/3338957198 wait complete. 2024-04-02T04:57:21.189 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.185+0000 7f6979d75700 1 Processor -- start 2024-04-02T04:57:21.190 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.185+0000 7f6979d75700 1 -- start start 2024-04-02T04:57:21.190 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.186+0000 7f6979d75700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f69740fbce0 0x7f697410c8a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:21.191 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.186+0000 7f6979d75700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f69740fcf90 0x7f697410cde0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:21.191 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.186+0000 7f6979d75700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f697410d3e0 con 0x7f69740fbce0 2024-04-02T04:57:21.191 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.186+0000 7f6979d75700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f697410dc30 con 0x7f69740fcf90 2024-04-02T04:57:21.191 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.186+0000 7f69737fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f69740fbce0 0x7f697410c8a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:21.191 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.186+0000 7f6972ffd700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f69740fcf90 0x7f697410cde0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:21.191 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.186+0000 7f6972ffd700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f69740fcf90 0x7f697410cde0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.152:45042/0 (socket says 172.21.15.152:45042) 2024-04-02T04:57:21.191 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.186+0000 7f6972ffd700 1 -- 172.21.15.152:0/2841016699 learned_addr learned my addr 172.21.15.152:0/2841016699 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:57:21.191 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.186+0000 7f6972ffd700 1 -- 172.21.15.152:0/2841016699 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f69740fbce0 msgr2=0x7f697410c8a0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:21.191 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.186+0000 7f6972ffd700 1 --2- 172.21.15.152:0/2841016699 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f69740fbce0 0x7f697410c8a0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:21.191 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.186+0000 7f6972ffd700 1 -- 172.21.15.152:0/2841016699 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6968005040 con 0x7f69740fcf90 2024-04-02T04:57:21.191 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.186+0000 7f69737fe700 1 --2- 172.21.15.152:0/2841016699 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f69740fbce0 0x7f697410c8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-02T04:57:21.191 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.187+0000 7f6972ffd700 1 --2- 172.21.15.152:0/2841016699 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f69740fcf90 0x7f697410cde0 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f6968013870 tx=0x7f6968009e30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:21.192 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.187+0000 7f695bfff700 1 -- 172.21.15.152:0/2841016699 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f6968021070 con 0x7f69740fcf90 2024-04-02T04:57:21.192 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.187+0000 7f6979d75700 1 -- 172.21.15.152:0/2841016699 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f697410deb0 con 0x7f69740fcf90 2024-04-02T04:57:21.192 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.187+0000 7f6979d75700 1 -- 172.21.15.152:0/2841016699 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f697410e3f0 con 0x7f69740fcf90 2024-04-02T04:57:21.192 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.187+0000 7f695bfff700 1 -- 172.21.15.152:0/2841016699 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f6968009430 con 0x7f69740fcf90 2024-04-02T04:57:21.192 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.187+0000 7f695bfff700 1 -- 172.21.15.152:0/2841016699 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f696800e070 con 0x7f69740fcf90 2024-04-02T04:57:21.192 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.188+0000 7f6979d75700 1 -- 172.21.15.152:0/2841016699 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f697405f5f0 con 0x7f69740fcf90 2024-04-02T04:57:21.193 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.189+0000 7f695bfff700 1 -- 172.21.15.152:0/2841016699 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f6968013af0 con 0x7f69740fcf90 2024-04-02T04:57:21.193 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.189+0000 7f695bfff700 1 --2- 172.21.15.152:0/2841016699 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f695c0692a0 0x7f695c06b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:21.193 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.190+0000 7f695bfff700 1 -- 172.21.15.152:0/2841016699 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7f69680883b0 con 0x7f69740fcf90 2024-04-02T04:57:21.193 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.190+0000 7f69737fe700 1 --2- 172.21.15.152:0/2841016699 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f695c0692a0 0x7f695c06b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:21.194 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.190+0000 7f69737fe700 1 --2- 172.21.15.152:0/2841016699 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f695c0692a0 0x7f695c06b750 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f6964009b80 tx=0x7f696400b040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:21.202 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.198+0000 7f695bfff700 1 -- 172.21.15.152:0/2841016699 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f69680581c0 con 0x7f69740fcf90 2024-04-02T04:57:21.478 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.474+0000 7f6979d75700 1 -- 172.21.15.152:0/2841016699 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "versions"} v 0) v1 -- 0x7f697404bed0 con 0x7f69740fcf90 2024-04-02T04:57:21.479 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:21 smithi152 ceph-mon[29073]: pgmap v219: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.8 KiB/s rd, 597 B/s wr, 5 op/s 2024-04-02T04:57:21.479 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:21 smithi152 ceph-mon[29073]: from='client.14576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:57:21.479 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.475+0000 7f695bfff700 1 -- 172.21.15.152:0/2841016699 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "versions"}]=0 v0) v1 ==== 56+0+638 (secure 0 0 0) 0x7f6968057d50 con 0x7f69740fcf90 2024-04-02T04:57:21.480 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-04-02T04:57:21.480 INFO:teuthology.orchestra.run.smithi152.stdout: "mon": { 2024-04-02T04:57:21.480 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 17.2.7-803-gc924f991 (c924f9914f2809997b363f77c5cbb14929881220) quincy (stable)": 2 2024-04-02T04:57:21.480 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-04-02T04:57:21.480 INFO:teuthology.orchestra.run.smithi152.stdout: "mgr": { 2024-04-02T04:57:21.480 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 17.2.7-803-gc924f991 (c924f9914f2809997b363f77c5cbb14929881220) quincy (stable)": 2 2024-04-02T04:57:21.480 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-04-02T04:57:21.480 INFO:teuthology.orchestra.run.smithi152.stdout: "osd": { 2024-04-02T04:57:21.480 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 17.2.7-803-gc924f991 (c924f9914f2809997b363f77c5cbb14929881220) quincy (stable)": 6 2024-04-02T04:57:21.480 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-04-02T04:57:21.480 INFO:teuthology.orchestra.run.smithi152.stdout: "mds": { 2024-04-02T04:57:21.480 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 17.2.7-803-gc924f991 (c924f9914f2809997b363f77c5cbb14929881220) quincy (stable)": 4 2024-04-02T04:57:21.481 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-04-02T04:57:21.481 INFO:teuthology.orchestra.run.smithi152.stdout: "overall": { 2024-04-02T04:57:21.481 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 17.2.7-803-gc924f991 (c924f9914f2809997b363f77c5cbb14929881220) quincy (stable)": 14 2024-04-02T04:57:21.481 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-04-02T04:57:21.481 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-04-02T04:57:21.482 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.478+0000 7f6979d75700 1 -- 172.21.15.152:0/2841016699 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f695c0692a0 msgr2=0x7f695c06b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:21.482 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.478+0000 7f6979d75700 1 --2- 172.21.15.152:0/2841016699 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f695c0692a0 0x7f695c06b750 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 crypto rx=0x7f6964009b80 tx=0x7f696400b040 comp rx=0 tx=0).stop 2024-04-02T04:57:21.482 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.478+0000 7f6979d75700 1 -- 172.21.15.152:0/2841016699 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f69740fcf90 msgr2=0x7f697410cde0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:21.482 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.478+0000 7f6979d75700 1 --2- 172.21.15.152:0/2841016699 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f69740fcf90 0x7f697410cde0 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 crypto rx=0x7f6968013870 tx=0x7f6968009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:21.483 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.479+0000 7f6979d75700 1 -- 172.21.15.152:0/2841016699 shutdown_connections 2024-04-02T04:57:21.483 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.479+0000 7f6979d75700 1 --2- 172.21.15.152:0/2841016699 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f695c0692a0 0x7f695c06b750 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:21.483 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.479+0000 7f6979d75700 1 --2- 172.21.15.152:0/2841016699 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f69740fbce0 0x7f697410c8a0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:21.483 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.479+0000 7f6979d75700 1 --2- 172.21.15.152:0/2841016699 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f69740fcf90 0x7f697410cde0 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:21.483 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.479+0000 7f6979d75700 1 -- 172.21.15.152:0/2841016699 >> 172.21.15.152:0/2841016699 conn(0x7f69740f7480 msgr2=0x7f69740f8aa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:21.483 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.479+0000 7f6979d75700 1 -- 172.21.15.152:0/2841016699 shutdown_connections 2024-04-02T04:57:21.483 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.479+0000 7f6979d75700 1 -- 172.21.15.152:0/2841016699 wait complete. 2024-04-02T04:57:21.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:21 smithi154 ceph-mon[35729]: pgmap v219: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.8 KiB/s rd, 597 B/s wr, 5 op/s 2024-04-02T04:57:21.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:21 smithi154 ceph-mon[35729]: from='client.14576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:57:21.591 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.587+0000 7f11bd1a4700 1 -- 172.21.15.152:0/3225726634 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f11b80fdff0 msgr2=0x7f11b80fe460 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:21.591 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.587+0000 7f11bd1a4700 1 --2- 172.21.15.152:0/3225726634 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f11b80fdff0 0x7f11b80fe460 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 crypto rx=0x7f11ac004940 tx=0x7f11ac009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:21.591 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.587+0000 7f11bd1a4700 1 -- 172.21.15.152:0/3225726634 shutdown_connections 2024-04-02T04:57:21.591 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.587+0000 7f11bd1a4700 1 --2- 172.21.15.152:0/3225726634 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f11b80fdff0 0x7f11b80fe460 unknown :-1 s=CLOSED pgs=297 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:21.591 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.587+0000 7f11bd1a4700 1 --2- 172.21.15.152:0/3225726634 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f11b80fce80 0x7f11b80fd290 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:21.591 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.587+0000 7f11bd1a4700 1 -- 172.21.15.152:0/3225726634 >> 172.21.15.152:0/3225726634 conn(0x7f11b80f8580 msgr2=0x7f11b80fa990 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:21.591 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.587+0000 7f11bd1a4700 1 -- 172.21.15.152:0/3225726634 shutdown_connections 2024-04-02T04:57:21.591 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.587+0000 7f11bd1a4700 1 -- 172.21.15.152:0/3225726634 wait complete. 2024-04-02T04:57:21.593 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.588+0000 7f11bd1a4700 1 Processor -- start 2024-04-02T04:57:21.593 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.588+0000 7f11bd1a4700 1 -- start start 2024-04-02T04:57:21.593 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.588+0000 7f11bd1a4700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f11b80fce80 0x7f11b8071dd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:21.593 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.588+0000 7f11bd1a4700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f11b80fdff0 0x7f11b8070310 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:21.593 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.588+0000 7f11bd1a4700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f11b8111680 con 0x7f11b80fce80 2024-04-02T04:57:21.593 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.588+0000 7f11bd1a4700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f11b81117c0 con 0x7f11b80fdff0 2024-04-02T04:57:21.593 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.588+0000 7f11b6d9d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f11b80fce80 0x7f11b8071dd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:21.593 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.588+0000 7f11b659c700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f11b80fdff0 0x7f11b8070310 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:21.594 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.588+0000 7f11b659c700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f11b80fdff0 0x7f11b8070310 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.152:45066/0 (socket says 172.21.15.152:45066) 2024-04-02T04:57:21.594 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.588+0000 7f11b659c700 1 -- 172.21.15.152:0/2759541153 learned_addr learned my addr 172.21.15.152:0/2759541153 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:57:21.594 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.589+0000 7f11b659c700 1 -- 172.21.15.152:0/2759541153 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f11b80fce80 msgr2=0x7f11b8071dd0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:21.594 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.589+0000 7f11b659c700 1 --2- 172.21.15.152:0/2759541153 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f11b80fce80 0x7f11b8071dd0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:21.594 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.589+0000 7f11b659c700 1 -- 172.21.15.152:0/2759541153 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f11ac005040 con 0x7f11b80fdff0 2024-04-02T04:57:21.594 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.589+0000 7f11b659c700 1 --2- 172.21.15.152:0/2759541153 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f11b80fdff0 0x7f11b8070310 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f11ac00b3b0 tx=0x7f11ac009e30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:21.594 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.589+0000 7f119f7fe700 1 -- 172.21.15.152:0/2759541153 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f11ac00e040 con 0x7f11b80fdff0 2024-04-02T04:57:21.594 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.589+0000 7f11bd1a4700 1 -- 172.21.15.152:0/2759541153 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f11b80708b0 con 0x7f11b80fdff0 2024-04-02T04:57:21.594 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.589+0000 7f119f7fe700 1 -- 172.21.15.152:0/2759541153 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f11ac00bbf0 con 0x7f11b80fdff0 2024-04-02T04:57:21.594 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.589+0000 7f119f7fe700 1 -- 172.21.15.152:0/2759541153 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f11ac01c630 con 0x7f11b80fdff0 2024-04-02T04:57:21.594 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.589+0000 7f11bd1a4700 1 -- 172.21.15.152:0/2759541153 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f11b8070e70 con 0x7f11b80fdff0 2024-04-02T04:57:21.595 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.590+0000 7f11bd1a4700 1 -- 172.21.15.152:0/2759541153 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f11b805f5f0 con 0x7f11b80fdff0 2024-04-02T04:57:21.595 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.591+0000 7f119f7fe700 1 -- 172.21.15.152:0/2759541153 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f11ac013420 con 0x7f11b80fdff0 2024-04-02T04:57:21.595 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.592+0000 7f119f7fe700 1 --2- 172.21.15.152:0/2759541153 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f11a00692a0 0x7f11a006b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:21.596 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.592+0000 7f119f7fe700 1 -- 172.21.15.152:0/2759541153 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7f11ac087df0 con 0x7f11b80fdff0 2024-04-02T04:57:21.596 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.592+0000 7f11b6d9d700 1 --2- 172.21.15.152:0/2759541153 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f11a00692a0 0x7f11a006b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:21.596 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.592+0000 7f11b6d9d700 1 --2- 172.21.15.152:0/2759541153 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f11a00692a0 0x7f11a006b750 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f11b8071b60 tx=0x7f11a800b040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:21.603 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.599+0000 7f119f7fe700 1 -- 172.21.15.152:0/2759541153 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f11ac054170 con 0x7f11b80fdff0 2024-04-02T04:57:21.846 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.842+0000 7f11bd1a4700 1 -- 172.21.15.152:0/2759541153 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "fs dump"} v 0) v1 -- 0x7f11b804bed0 con 0x7f11b80fdff0 2024-04-02T04:57:21.847 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.843+0000 7f119f7fe700 1 -- 172.21.15.152:0/2759541153 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump"}]=0 dumped fsmap epoch 20 v20) v1 ==== 76+0+1833 (secure 0 0 0) 0x7f11ac057790 con 0x7f11b80fdff0 2024-04-02T04:57:21.847 INFO:teuthology.orchestra.run.smithi152.stdout:e20 2024-04-02T04:57:21.847 INFO:teuthology.orchestra.run.smithi152.stdout:enable_multiple, ever_enabled_multiple: 1,1 2024-04-02T04:57:21.847 INFO:teuthology.orchestra.run.smithi152.stdout:default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-02T04:57:21.847 INFO:teuthology.orchestra.run.smithi152.stdout:legacy client fscid: 1 2024-04-02T04:57:21.847 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:57:21.847 INFO:teuthology.orchestra.run.smithi152.stdout:Filesystem 'cephfs' (1) 2024-04-02T04:57:21.847 INFO:teuthology.orchestra.run.smithi152.stdout:fs_name cephfs 2024-04-02T04:57:21.848 INFO:teuthology.orchestra.run.smithi152.stdout:epoch 20 2024-04-02T04:57:21.848 INFO:teuthology.orchestra.run.smithi152.stdout:flags 32 joinable allow_snaps allow_multimds_snaps allow_standby_replay 2024-04-02T04:57:21.848 INFO:teuthology.orchestra.run.smithi152.stdout:created 2024-04-02T04:55:50.446442+0000 2024-04-02T04:57:21.848 INFO:teuthology.orchestra.run.smithi152.stdout:modified 2024-04-02T04:56:08.533665+0000 2024-04-02T04:57:21.848 INFO:teuthology.orchestra.run.smithi152.stdout:tableserver 0 2024-04-02T04:57:21.848 INFO:teuthology.orchestra.run.smithi152.stdout:root 0 2024-04-02T04:57:21.848 INFO:teuthology.orchestra.run.smithi152.stdout:session_timeout 60 2024-04-02T04:57:21.848 INFO:teuthology.orchestra.run.smithi152.stdout:session_autoclose 300 2024-04-02T04:57:21.848 INFO:teuthology.orchestra.run.smithi152.stdout:max_file_size 1099511627776 2024-04-02T04:57:21.848 INFO:teuthology.orchestra.run.smithi152.stdout:required_client_features {} 2024-04-02T04:57:21.848 INFO:teuthology.orchestra.run.smithi152.stdout:last_failure 0 2024-04-02T04:57:21.848 INFO:teuthology.orchestra.run.smithi152.stdout:last_failure_osd_epoch 46 2024-04-02T04:57:21.848 INFO:teuthology.orchestra.run.smithi152.stdout:compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-02T04:57:21.848 INFO:teuthology.orchestra.run.smithi152.stdout:max_mds 2 2024-04-02T04:57:21.848 INFO:teuthology.orchestra.run.smithi152.stdout:in 0,1 2024-04-02T04:57:21.849 INFO:teuthology.orchestra.run.smithi152.stdout:up {0=14456,1=14470} 2024-04-02T04:57:21.849 INFO:teuthology.orchestra.run.smithi152.stdout:failed 2024-04-02T04:57:21.849 INFO:teuthology.orchestra.run.smithi152.stdout:damaged 2024-04-02T04:57:21.849 INFO:teuthology.orchestra.run.smithi152.stdout:stopped 2024-04-02T04:57:21.849 INFO:teuthology.orchestra.run.smithi152.stdout:data_pools [3] 2024-04-02T04:57:21.849 INFO:teuthology.orchestra.run.smithi152.stdout:metadata_pool 2 2024-04-02T04:57:21.849 INFO:teuthology.orchestra.run.smithi152.stdout:inline_data enabled 2024-04-02T04:57:21.849 INFO:teuthology.orchestra.run.smithi152.stdout:balancer 2024-04-02T04:57:21.849 INFO:teuthology.orchestra.run.smithi152.stdout:standby_count_wanted 1 2024-04-02T04:57:21.849 INFO:teuthology.orchestra.run.smithi152.stdout:[mds.cephfs.smithi152.lluqrs{0:14456} state up:active seq 5 join_fscid=1 addr [v2:172.21.15.152:6826/4038990242,v1:172.21.15.152:6827/4038990242] compat {c=[1],r=[1],i=[7ff]}] 2024-04-02T04:57:21.849 INFO:teuthology.orchestra.run.smithi152.stdout:[mds.cephfs.smithi154.rbkmwp{0:24297} state up:standby-replay seq 2 join_fscid=1 addr [v2:172.21.15.154:6826/53707828,v1:172.21.15.154:6827/53707828] compat {c=[1],r=[1],i=[7ff]}] 2024-04-02T04:57:21.849 INFO:teuthology.orchestra.run.smithi152.stdout:[mds.cephfs.smithi152.ywghzy{1:14470} state up:active seq 6 join_fscid=1 addr [v2:172.21.15.152:6828/3554217123,v1:172.21.15.152:6829/3554217123] compat {c=[1],r=[1],i=[7ff]}] 2024-04-02T04:57:21.849 INFO:teuthology.orchestra.run.smithi152.stdout:[mds.cephfs.smithi154.llvaoh{1:24301} state up:standby-replay seq 1 join_fscid=1 addr [v2:172.21.15.154:6824/1569612467,v1:172.21.15.154:6825/1569612467] compat {c=[1],r=[1],i=[7ff]}] 2024-04-02T04:57:21.850 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:57:21.850 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:57:21.850 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.846+0000 7f11bd1a4700 1 -- 172.21.15.152:0/2759541153 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f11a00692a0 msgr2=0x7f11a006b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:21.850 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.846+0000 7f11bd1a4700 1 --2- 172.21.15.152:0/2759541153 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f11a00692a0 0x7f11a006b750 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 crypto rx=0x7f11b8071b60 tx=0x7f11a800b040 comp rx=0 tx=0).stop 2024-04-02T04:57:21.851 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.846+0000 7f11bd1a4700 1 -- 172.21.15.152:0/2759541153 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f11b80fdff0 msgr2=0x7f11b8070310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:21.851 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.846+0000 7f11bd1a4700 1 --2- 172.21.15.152:0/2759541153 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f11b80fdff0 0x7f11b8070310 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 crypto rx=0x7f11ac00b3b0 tx=0x7f11ac009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:21.851 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.846+0000 7f11bd1a4700 1 -- 172.21.15.152:0/2759541153 shutdown_connections 2024-04-02T04:57:21.851 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.846+0000 7f11bd1a4700 1 --2- 172.21.15.152:0/2759541153 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f11a00692a0 0x7f11a006b750 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:21.851 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.846+0000 7f11bd1a4700 1 --2- 172.21.15.152:0/2759541153 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f11b80fce80 0x7f11b8071dd0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:21.851 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.846+0000 7f11bd1a4700 1 --2- 172.21.15.152:0/2759541153 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f11b80fdff0 0x7f11b8070310 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:21.851 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.846+0000 7f11bd1a4700 1 -- 172.21.15.152:0/2759541153 >> 172.21.15.152:0/2759541153 conn(0x7f11b80f8580 msgr2=0x7f11b80f9c50 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:21.851 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.846+0000 7f11bd1a4700 1 -- 172.21.15.152:0/2759541153 shutdown_connections 2024-04-02T04:57:21.851 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:21.846+0000 7f11bd1a4700 1 -- 172.21.15.152:0/2759541153 wait complete. 2024-04-02T04:57:21.851 INFO:teuthology.orchestra.run.smithi152.stderr:dumped fsmap epoch 20 2024-04-02T04:57:22.011 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.007+0000 7f827672f700 1 -- 172.21.15.152:0/1608779739 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f82700fbcd0 msgr2=0x7f82700fc0e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:22.011 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.007+0000 7f827672f700 1 --2- 172.21.15.152:0/1608779739 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f82700fbcd0 0x7f82700fc0e0 secure :-1 s=READY pgs=298 cs=0 l=1 rev1=1 crypto rx=0x7f8260004940 tx=0x7f8260009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:22.011 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.007+0000 7f827672f700 1 -- 172.21.15.152:0/1608779739 shutdown_connections 2024-04-02T04:57:22.011 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.007+0000 7f827672f700 1 --2- 172.21.15.152:0/1608779739 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f82700fcf80 0x7f82700fd3f0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:22.011 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.007+0000 7f827672f700 1 --2- 172.21.15.152:0/1608779739 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f82700fbcd0 0x7f82700fc0e0 unknown :-1 s=CLOSED pgs=298 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:22.011 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.007+0000 7f827672f700 1 -- 172.21.15.152:0/1608779739 >> 172.21.15.152:0/1608779739 conn(0x7f82700f7430 msgr2=0x7f82700f9860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:22.011 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.007+0000 7f827672f700 1 -- 172.21.15.152:0/1608779739 shutdown_connections 2024-04-02T04:57:22.012 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.007+0000 7f827672f700 1 -- 172.21.15.152:0/1608779739 wait complete. 2024-04-02T04:57:22.013 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.007+0000 7f827672f700 1 Processor -- start 2024-04-02T04:57:22.013 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.008+0000 7f827672f700 1 -- start start 2024-04-02T04:57:22.014 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.008+0000 7f827672f700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f82700fcf80 0x7f8270110890 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:22.014 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.008+0000 7f827672f700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8270066240 0x7f8270112de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:22.014 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.008+0000 7f827672f700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8270113320 con 0x7f8270066240 2024-04-02T04:57:22.014 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.008+0000 7f827672f700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8270113490 con 0x7f82700fcf80 2024-04-02T04:57:22.014 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.008+0000 7f826f7fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8270066240 0x7f8270112de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:22.014 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.008+0000 7f826f7fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8270066240 0x7f8270112de0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:42334/0 (socket says 172.21.15.152:42334) 2024-04-02T04:57:22.014 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.008+0000 7f826f7fe700 1 -- 172.21.15.152:0/1586344470 learned_addr learned my addr 172.21.15.152:0/1586344470 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:57:22.014 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.008+0000 7f826ffff700 1 --2- 172.21.15.152:0/1586344470 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f82700fcf80 0x7f8270110890 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:22.014 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.009+0000 7f826f7fe700 1 -- 172.21.15.152:0/1586344470 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f82700fcf80 msgr2=0x7f8270110890 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:22.014 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.009+0000 7f826f7fe700 1 --2- 172.21.15.152:0/1586344470 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f82700fcf80 0x7f8270110890 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:22.014 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.009+0000 7f826f7fe700 1 -- 172.21.15.152:0/1586344470 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8260005040 con 0x7f8270066240 2024-04-02T04:57:22.015 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.009+0000 7f826f7fe700 1 --2- 172.21.15.152:0/1586344470 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8270066240 0x7f8270112de0 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7f8264004ee0 tx=0x7f82640046b0 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:22.015 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.009+0000 7f826cff9700 1 -- 172.21.15.152:0/1586344470 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f8264007ea0 con 0x7f8270066240 2024-04-02T04:57:22.015 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.009+0000 7f826cff9700 1 -- 172.21.15.152:0/1586344470 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f82640095a0 con 0x7f8270066240 2024-04-02T04:57:22.015 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.009+0000 7f827672f700 1 -- 172.21.15.152:0/1586344470 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f827010c6d0 con 0x7f8270066240 2024-04-02T04:57:22.015 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.009+0000 7f827672f700 1 -- 172.21.15.152:0/1586344470 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f827010cc60 con 0x7f8270066240 2024-04-02T04:57:22.015 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.010+0000 7f826cff9700 1 -- 172.21.15.152:0/1586344470 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f826400ddd0 con 0x7f8270066240 2024-04-02T04:57:22.015 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.010+0000 7f827672f700 1 -- 172.21.15.152:0/1586344470 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f827005f5f0 con 0x7f8270066240 2024-04-02T04:57:22.017 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.014+0000 7f826cff9700 1 -- 172.21.15.152:0/1586344470 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f8264018070 con 0x7f8270066240 2024-04-02T04:57:22.018 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.014+0000 7f826cff9700 1 --2- 172.21.15.152:0/1586344470 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f8258069360 0x7f825806b810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:22.018 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.015+0000 7f826cff9700 1 -- 172.21.15.152:0/1586344470 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7f826408b2c0 con 0x7f8270066240 2024-04-02T04:57:22.019 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.015+0000 7f826ffff700 1 --2- 172.21.15.152:0/1586344470 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f8258069360 0x7f825806b810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:22.019 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.016+0000 7f826ffff700 1 --2- 172.21.15.152:0/1586344470 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f8258069360 0x7f825806b810 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f8260000c00 tx=0x7f8260009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:22.025 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.021+0000 7f826cff9700 1 -- 172.21.15.152:0/1586344470 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f8264057500 con 0x7f8270066240 2024-04-02T04:57:22.315 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.311+0000 7f827672f700 1 -- 172.21.15.152:0/1586344470 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}) v1 -- 0x7f8270101420 con 0x7f8258069360 2024-04-02T04:57:22.316 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.313+0000 7f826cff9700 1 -- 172.21.15.152:0/1586344470 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+447 (secure 0 0 0) 0x7f8270101420 con 0x7f8258069360 2024-04-02T04:57:22.317 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-04-02T04:57:22.317 INFO:teuthology.orchestra.run.smithi152.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae", 2024-04-02T04:57:22.317 INFO:teuthology.orchestra.run.smithi152.stdout: "in_progress": true, 2024-04-02T04:57:22.317 INFO:teuthology.orchestra.run.smithi152.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-02T04:57:22.317 INFO:teuthology.orchestra.run.smithi152.stdout: "services_complete": [], 2024-04-02T04:57:22.317 INFO:teuthology.orchestra.run.smithi152.stdout: "progress": "0/21 daemons upgraded", 2024-04-02T04:57:22.317 INFO:teuthology.orchestra.run.smithi152.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae image on host smithi154", 2024-04-02T04:57:22.317 INFO:teuthology.orchestra.run.smithi152.stdout: "is_paused": false 2024-04-02T04:57:22.317 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-04-02T04:57:22.319 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.315+0000 7f827672f700 1 -- 172.21.15.152:0/1586344470 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f8258069360 msgr2=0x7f825806b810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:22.319 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.315+0000 7f827672f700 1 --2- 172.21.15.152:0/1586344470 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f8258069360 0x7f825806b810 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 crypto rx=0x7f8260000c00 tx=0x7f8260009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:22.319 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.315+0000 7f827672f700 1 -- 172.21.15.152:0/1586344470 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8270066240 msgr2=0x7f8270112de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:22.319 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.315+0000 7f827672f700 1 --2- 172.21.15.152:0/1586344470 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8270066240 0x7f8270112de0 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 crypto rx=0x7f8264004ee0 tx=0x7f82640046b0 comp rx=0 tx=0).stop 2024-04-02T04:57:22.319 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.315+0000 7f827672f700 1 -- 172.21.15.152:0/1586344470 shutdown_connections 2024-04-02T04:57:22.319 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.315+0000 7f827672f700 1 --2- 172.21.15.152:0/1586344470 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f8258069360 0x7f825806b810 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:22.320 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.315+0000 7f827672f700 1 --2- 172.21.15.152:0/1586344470 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f82700fcf80 0x7f8270110890 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:22.320 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.315+0000 7f827672f700 1 --2- 172.21.15.152:0/1586344470 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f8270066240 0x7f8270112de0 unknown :-1 s=CLOSED pgs=299 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:22.320 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.315+0000 7f827672f700 1 -- 172.21.15.152:0/1586344470 >> 172.21.15.152:0/1586344470 conn(0x7f82700f7430 msgr2=0x7f82700f8bf0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:22.320 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.316+0000 7f827672f700 1 -- 172.21.15.152:0/1586344470 shutdown_connections 2024-04-02T04:57:22.320 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.316+0000 7f827672f700 1 -- 172.21.15.152:0/1586344470 wait complete. 2024-04-02T04:57:22.425 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.421+0000 7fb6143cb700 1 -- 172.21.15.152:0/4077180963 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb60c066b10 msgr2=0x7fb60c0ffa40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:22.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.421+0000 7fb6143cb700 1 --2- 172.21.15.152:0/4077180963 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb60c066b10 0x7fb60c0ffa40 secure :-1 s=READY pgs=300 cs=0 l=1 rev1=1 crypto rx=0x7fb608004940 tx=0x7fb608009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:22.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.422+0000 7fb6143cb700 1 -- 172.21.15.152:0/4077180963 shutdown_connections 2024-04-02T04:57:22.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.422+0000 7fb6143cb700 1 --2- 172.21.15.152:0/4077180963 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb60c0fff80 0x7fb60c102360 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:22.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.422+0000 7fb6143cb700 1 --2- 172.21.15.152:0/4077180963 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb60c066b10 0x7fb60c0ffa40 unknown :-1 s=CLOSED pgs=300 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:22.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.422+0000 7fb6143cb700 1 -- 172.21.15.152:0/4077180963 >> 172.21.15.152:0/4077180963 conn(0x7fb60c0f7450 msgr2=0x7fb60c0f9880 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:22.426 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:22 smithi152 ceph-mon[29073]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:57:22.426 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:22 smithi152 ceph-mon[29073]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:57:22.427 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:22 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/2841016699' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T04:57:22.427 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:22 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/2759541153' entity='client.admin' cmd=[{"prefix": "fs dump"}]: dispatch 2024-04-02T04:57:22.427 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.422+0000 7fb6143cb700 1 -- 172.21.15.152:0/4077180963 shutdown_connections 2024-04-02T04:57:22.428 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.425+0000 7fb6143cb700 1 -- 172.21.15.152:0/4077180963 wait complete. 2024-04-02T04:57:22.429 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.425+0000 7fb6143cb700 1 Processor -- start 2024-04-02T04:57:22.429 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.425+0000 7fb6143cb700 1 -- start start 2024-04-02T04:57:22.430 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.426+0000 7fb6143cb700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb60c066b10 0x7fb60c10da60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:22.430 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.426+0000 7fb6143cb700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb60c0fff80 0x7fb60c10dfa0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:22.430 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.426+0000 7fb6143cb700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb60c10e5a0 con 0x7fb60c066b10 2024-04-02T04:57:22.430 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.426+0000 7fb6143cb700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb60c10eda0 con 0x7fb60c0fff80 2024-04-02T04:57:22.430 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.426+0000 7fb612167700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb60c066b10 0x7fb60c10da60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:22.430 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.426+0000 7fb612167700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb60c066b10 0x7fb60c10da60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:42352/0 (socket says 172.21.15.152:42352) 2024-04-02T04:57:22.430 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.426+0000 7fb612167700 1 -- 172.21.15.152:0/973165492 learned_addr learned my addr 172.21.15.152:0/973165492 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:57:22.430 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.426+0000 7fb611966700 1 --2- 172.21.15.152:0/973165492 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb60c0fff80 0x7fb60c10dfa0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:22.431 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.426+0000 7fb612167700 1 -- 172.21.15.152:0/973165492 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb60c0fff80 msgr2=0x7fb60c10dfa0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:22.431 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.426+0000 7fb612167700 1 --2- 172.21.15.152:0/973165492 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb60c0fff80 0x7fb60c10dfa0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:22.431 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.426+0000 7fb612167700 1 -- 172.21.15.152:0/973165492 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb608005040 con 0x7fb60c066b10 2024-04-02T04:57:22.431 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.427+0000 7fb612167700 1 --2- 172.21.15.152:0/973165492 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb60c066b10 0x7fb60c10da60 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7fb608000c00 tx=0x7fb608009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:22.432 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.427+0000 7fb602ffd700 1 -- 172.21.15.152:0/973165492 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fb60800e030 con 0x7fb60c066b10 2024-04-02T04:57:22.432 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.427+0000 7fb6143cb700 1 -- 172.21.15.152:0/973165492 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb60c10f020 con 0x7fb60c066b10 2024-04-02T04:57:22.432 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.427+0000 7fb6143cb700 1 -- 172.21.15.152:0/973165492 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb60c10f560 con 0x7fb60c066b10 2024-04-02T04:57:22.432 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.427+0000 7fb602ffd700 1 -- 172.21.15.152:0/973165492 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fb608013de0 con 0x7fb60c066b10 2024-04-02T04:57:22.432 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.427+0000 7fb602ffd700 1 -- 172.21.15.152:0/973165492 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fb60801ce50 con 0x7fb60c066b10 2024-04-02T04:57:22.432 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.428+0000 7fb600ff9700 1 -- 172.21.15.152:0/973165492 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb60c05f5f0 con 0x7fb60c066b10 2024-04-02T04:57:22.434 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.429+0000 7fb602ffd700 1 -- 172.21.15.152:0/973165492 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7fb60801c400 con 0x7fb60c066b10 2024-04-02T04:57:22.434 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.430+0000 7fb602ffd700 1 --2- 172.21.15.152:0/973165492 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb5f806d6b0 0x7fb5f806fb60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:22.434 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.430+0000 7fb602ffd700 1 -- 172.21.15.152:0/973165492 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7fb608088ba0 con 0x7fb60c066b10 2024-04-02T04:57:22.434 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.431+0000 7fb611966700 1 --2- 172.21.15.152:0/973165492 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb5f806d6b0 0x7fb5f806fb60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:22.435 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.431+0000 7fb611966700 1 --2- 172.21.15.152:0/973165492 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb5f806d6b0 0x7fb5f806fb60 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7fb5fc009b80 tx=0x7fb5fc00b040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:22.438 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.434+0000 7fb602ffd700 1 -- 172.21.15.152:0/973165492 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fb608058910 con 0x7fb60c066b10 2024-04-02T04:57:22.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:22 smithi154 ceph-mon[35729]: from='client.14580 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:57:22.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:22 smithi154 ceph-mon[35729]: from='client.14584 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:57:22.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:22 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/2841016699' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T04:57:22.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:22 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/2759541153' entity='client.admin' cmd=[{"prefix": "fs dump"}]: dispatch 2024-04-02T04:57:22.713 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.709+0000 7fb600ff9700 1 -- 172.21.15.152:0/973165492 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "health", "detail": "detail"} v 0) v1 -- 0x7fb60c05f5f0 con 0x7fb60c066b10 2024-04-02T04:57:22.714 INFO:teuthology.orchestra.run.smithi152.stdout:HEALTH_WARN 1 filesystem with deprecated feature inline_data 2024-04-02T04:57:22.714 INFO:teuthology.orchestra.run.smithi152.stdout:[WRN] FS_INLINE_DATA_DEPRECATED: 1 filesystem with deprecated feature inline_data 2024-04-02T04:57:22.714 INFO:teuthology.orchestra.run.smithi152.stdout: fs cephfs has deprecated feature inline_data enabled. 2024-04-02T04:57:22.714 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.709+0000 7fb602ffd700 1 -- 172.21.15.152:0/973165492 <== mon.0 v2:172.21.15.152:3300/0 7 ==== mon_command_ack([{"prefix": "health", "detail": "detail"}]=0 v0) v1 ==== 74+0+201 (secure 0 0 0) 0x7fb6080584a0 con 0x7fb60c066b10 2024-04-02T04:57:22.717 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.712+0000 7fb600ff9700 1 -- 172.21.15.152:0/973165492 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb5f806d6b0 msgr2=0x7fb5f806fb60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:22.717 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.712+0000 7fb600ff9700 1 --2- 172.21.15.152:0/973165492 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb5f806d6b0 0x7fb5f806fb60 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 crypto rx=0x7fb5fc009b80 tx=0x7fb5fc00b040 comp rx=0 tx=0).stop 2024-04-02T04:57:22.717 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.712+0000 7fb600ff9700 1 -- 172.21.15.152:0/973165492 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb60c066b10 msgr2=0x7fb60c10da60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:22.717 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.712+0000 7fb600ff9700 1 --2- 172.21.15.152:0/973165492 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb60c066b10 0x7fb60c10da60 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 crypto rx=0x7fb608000c00 tx=0x7fb608009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:22.717 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.713+0000 7fb600ff9700 1 -- 172.21.15.152:0/973165492 shutdown_connections 2024-04-02T04:57:22.717 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.713+0000 7fb600ff9700 1 --2- 172.21.15.152:0/973165492 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb5f806d6b0 0x7fb5f806fb60 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:22.717 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.713+0000 7fb600ff9700 1 --2- 172.21.15.152:0/973165492 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb60c066b10 0x7fb60c10da60 unknown :-1 s=CLOSED pgs=301 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:22.717 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.713+0000 7fb600ff9700 1 --2- 172.21.15.152:0/973165492 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb60c0fff80 0x7fb60c10dfa0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:22.717 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.713+0000 7fb600ff9700 1 -- 172.21.15.152:0/973165492 >> 172.21.15.152:0/973165492 conn(0x7fb60c0f7450 msgr2=0x7fb60c0f7870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:22.717 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.713+0000 7fb600ff9700 1 -- 172.21.15.152:0/973165492 shutdown_connections 2024-04-02T04:57:22.718 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:22.713+0000 7fb600ff9700 1 -- 172.21.15.152:0/973165492 wait complete. 2024-04-02T04:57:23.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:23 smithi154 ceph-mon[35729]: pgmap v220: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 170 B/s wr, 4 op/s 2024-04-02T04:57:23.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:23 smithi154 ceph-mon[35729]: from='client.14596 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:57:23.484 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:23 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/973165492' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T04:57:23.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:23 smithi152 ceph-mon[29073]: pgmap v220: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 170 B/s wr, 4 op/s 2024-04-02T04:57:23.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:23 smithi152 ceph-mon[29073]: from='client.14596 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:57:23.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:23 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/973165492' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T04:57:25.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:25 smithi154 ceph-mon[35729]: pgmap v221: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.5 KiB/s rd, 85 B/s wr, 4 op/s 2024-04-02T04:57:25.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:25 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:57:25.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:25 smithi152 ceph-mon[29073]: pgmap v221: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.5 KiB/s rd, 85 B/s wr, 4 op/s 2024-04-02T04:57:25.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:25 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:57:27.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:27 smithi154 ceph-mon[35729]: pgmap v222: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 85 B/s wr, 3 op/s 2024-04-02T04:57:27.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:27 smithi152 ceph-mon[29073]: pgmap v222: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 85 B/s wr, 3 op/s 2024-04-02T04:57:29.431 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:29 smithi152 ceph-mon[29073]: pgmap v223: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 85 B/s wr, 3 op/s 2024-04-02T04:57:29.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:29 smithi154 ceph-mon[35729]: pgmap v223: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 85 B/s wr, 3 op/s 2024-04-02T04:57:31.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:31 smithi154 ceph-mon[35729]: pgmap v224: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-02T04:57:31.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:31 smithi152 ceph-mon[29073]: pgmap v224: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-02T04:57:33.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:33 smithi154 ceph-mon[35729]: pgmap v225: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 3 op/s 2024-04-02T04:57:33.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:33 smithi152 ceph-mon[29073]: pgmap v225: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 3 op/s 2024-04-02T04:57:35.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:35 smithi154 ceph-mon[35729]: pgmap v226: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-02T04:57:35.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:35 smithi152 ceph-mon[29073]: pgmap v226: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-02T04:57:37.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:37 smithi154 ceph-mon[35729]: pgmap v227: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 3 op/s 2024-04-02T04:57:37.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:37 smithi152 ceph-mon[29073]: pgmap v227: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 3 op/s 2024-04-02T04:57:39.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:39 smithi154 ceph-mon[35729]: pgmap v228: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 3 op/s 2024-04-02T04:57:39.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:39 smithi152 ceph-mon[29073]: pgmap v228: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 3 op/s 2024-04-02T04:57:39.929 INFO:tasks.workunit.client.1.smithi154.stderr:Updating files: 94% (11710/12435) Updating files: 95% (11814/12435) Updating files: 96% (11938/12435) Updating files: 97% (12062/12435) Updating files: 98% (12187/12435) Updating files: 99% (12311/12435) Updating files: 100% (12435/12435) Updating files: 100% (12435/12435), done. 2024-04-02T04:57:40.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:40 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:57:40.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:40 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:57:40.802 INFO:tasks.workunit.client.1.smithi154.stderr:Note: switching to '6cec793d9073f0594e2364b9b687ecaf22424cca'. 2024-04-02T04:57:40.802 INFO:tasks.workunit.client.1.smithi154.stderr: 2024-04-02T04:57:40.802 INFO:tasks.workunit.client.1.smithi154.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2024-04-02T04:57:40.802 INFO:tasks.workunit.client.1.smithi154.stderr:changes and commit them, and you can discard any commits you make in this 2024-04-02T04:57:40.802 INFO:tasks.workunit.client.1.smithi154.stderr:state without impacting any branches by switching back to a branch. 2024-04-02T04:57:40.802 INFO:tasks.workunit.client.1.smithi154.stderr: 2024-04-02T04:57:40.802 INFO:tasks.workunit.client.1.smithi154.stderr:If you want to create a new branch to retain commits you create, you may 2024-04-02T04:57:40.802 INFO:tasks.workunit.client.1.smithi154.stderr:do so (now or later) by using -c with the switch command. Example: 2024-04-02T04:57:40.802 INFO:tasks.workunit.client.1.smithi154.stderr: 2024-04-02T04:57:40.802 INFO:tasks.workunit.client.1.smithi154.stderr: git switch -c 2024-04-02T04:57:40.802 INFO:tasks.workunit.client.1.smithi154.stderr: 2024-04-02T04:57:40.802 INFO:tasks.workunit.client.1.smithi154.stderr:Or undo this operation with: 2024-04-02T04:57:40.803 INFO:tasks.workunit.client.1.smithi154.stderr: 2024-04-02T04:57:40.803 INFO:tasks.workunit.client.1.smithi154.stderr: git switch - 2024-04-02T04:57:40.803 INFO:tasks.workunit.client.1.smithi154.stderr: 2024-04-02T04:57:40.803 INFO:tasks.workunit.client.1.smithi154.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2024-04-02T04:57:40.803 INFO:tasks.workunit.client.1.smithi154.stderr: 2024-04-02T04:57:40.803 INFO:tasks.workunit.client.1.smithi154.stderr:HEAD is now at 6cec793d907 Merge pull request #56601 from zdover23/wip-doc-2024-04-01-backport-56600-to-squid 2024-04-02T04:57:40.806 DEBUG:teuthology.orchestra.run.smithi154:> cd -- /home/ubuntu/cephtest/clone.client.1/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.1 2024-04-02T04:57:40.868 INFO:tasks.workunit.client.1.smithi154.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-04-02T04:57:40.870 INFO:tasks.workunit.client.1.smithi154.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.1/qa/workunits/direct_io' 2024-04-02T04:57:40.870 INFO:tasks.workunit.client.1.smithi154.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-04-02T04:57:41.173 INFO:tasks.workunit.client.0.smithi152.stderr:Updating files: 99% (12404/12435) Updating files: 100% (12435/12435) Updating files: 100% (12435/12435), done. 2024-04-02T04:57:41.453 INFO:tasks.workunit.client.0.smithi152.stderr:Note: switching to '6cec793d9073f0594e2364b9b687ecaf22424cca'. 2024-04-02T04:57:41.453 INFO:tasks.workunit.client.0.smithi152.stderr: 2024-04-02T04:57:41.453 INFO:tasks.workunit.client.0.smithi152.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2024-04-02T04:57:41.453 INFO:tasks.workunit.client.0.smithi152.stderr:changes and commit them, and you can discard any commits you make in this 2024-04-02T04:57:41.453 INFO:tasks.workunit.client.0.smithi152.stderr:state without impacting any branches by switching back to a branch. 2024-04-02T04:57:41.453 INFO:tasks.workunit.client.0.smithi152.stderr: 2024-04-02T04:57:41.453 INFO:tasks.workunit.client.0.smithi152.stderr:If you want to create a new branch to retain commits you create, you may 2024-04-02T04:57:41.453 INFO:tasks.workunit.client.0.smithi152.stderr:do so (now or later) by using -c with the switch command. Example: 2024-04-02T04:57:41.453 INFO:tasks.workunit.client.0.smithi152.stderr: 2024-04-02T04:57:41.453 INFO:tasks.workunit.client.0.smithi152.stderr: git switch -c 2024-04-02T04:57:41.453 INFO:tasks.workunit.client.0.smithi152.stderr: 2024-04-02T04:57:41.453 INFO:tasks.workunit.client.0.smithi152.stderr:Or undo this operation with: 2024-04-02T04:57:41.453 INFO:tasks.workunit.client.0.smithi152.stderr: 2024-04-02T04:57:41.454 INFO:tasks.workunit.client.0.smithi152.stderr: git switch - 2024-04-02T04:57:41.454 INFO:tasks.workunit.client.0.smithi152.stderr: 2024-04-02T04:57:41.454 INFO:tasks.workunit.client.0.smithi152.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2024-04-02T04:57:41.454 INFO:tasks.workunit.client.0.smithi152.stderr: 2024-04-02T04:57:41.454 INFO:tasks.workunit.client.0.smithi152.stderr:HEAD is now at 6cec793d907 Merge pull request #56601 from zdover23/wip-doc-2024-04-01-backport-56600-to-squid 2024-04-02T04:57:41.456 DEBUG:teuthology.orchestra.run.smithi152:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-04-02T04:57:41.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:41 smithi154 ceph-mon[35729]: pgmap v229: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-02T04:57:41.506 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:41 smithi152 ceph-mon[29073]: pgmap v229: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.5 KiB/s rd, 4 op/s 2024-04-02T04:57:41.522 INFO:tasks.workunit.client.0.smithi152.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-04-02T04:57:41.524 INFO:tasks.workunit.client.0.smithi152.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-04-02T04:57:41.524 INFO:tasks.workunit.client.0.smithi152.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-04-02T04:57:41.615 INFO:tasks.workunit.client.1.smithi154.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-04-02T04:57:41.685 INFO:tasks.workunit.client.1.smithi154.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-04-02T04:57:41.727 INFO:tasks.workunit.client.1.smithi154.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.1/qa/workunits/direct_io' 2024-04-02T04:57:41.729 INFO:tasks.workunit.client.1.smithi154.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.1/qa/workunits/fs' 2024-04-02T04:57:41.729 INFO:tasks.workunit.client.1.smithi154.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-04-02T04:57:41.769 INFO:tasks.workunit.client.1.smithi154.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.1/qa/workunits/fs' 2024-04-02T04:57:41.772 DEBUG:teuthology.orchestra.run.smithi154:> set -ex 2024-04-02T04:57:41.772 DEBUG:teuthology.orchestra.run.smithi154:> dd if=/home/ubuntu/cephtest/workunits.list.client.1 of=/dev/stdout 2024-04-02T04:57:41.828 INFO:tasks.workunit:Running workunits matching suites/fsstress.sh on client.1... 2024-04-02T04:57:41.830 INFO:tasks.workunit:Running workunit suites/fsstress.sh... 2024-04-02T04:57:41.831 DEBUG:teuthology.orchestra.run.smithi154:workunit test suites/fsstress.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=6cec793d9073f0594e2364b9b687ecaf22424cca TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 CEPH_MNT=/home/ubuntu/cephtest/mnt.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/fsstress.sh 2024-04-02T04:57:41.890 INFO:tasks.workunit.client.1.smithi154.stderr:+ mkdir -p fsstress 2024-04-02T04:57:41.891 INFO:tasks.workunit.client.1.smithi154.stderr:+ pushd fsstress 2024-04-02T04:57:41.891 INFO:tasks.workunit.client.1.smithi154.stdout:~/cephtest/mnt.1/client.1/tmp/fsstress ~/cephtest/mnt.1/client.1/tmp 2024-04-02T04:57:41.891 INFO:tasks.workunit.client.1.smithi154.stderr:+ wget -q -O ltp-full.tgz http://download.ceph.com/qa/ltp-full-20091231.tgz 2024-04-02T04:57:42.346 INFO:tasks.workunit.client.0.smithi152.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-04-02T04:57:42.398 INFO:tasks.workunit.client.0.smithi152.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-04-02T04:57:42.426 INFO:tasks.workunit.client.1.smithi154.stderr:+ tar xzf ltp-full.tgz 2024-04-02T04:57:42.437 INFO:tasks.workunit.client.0.smithi152.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-04-02T04:57:42.439 INFO:tasks.workunit.client.0.smithi152.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-04-02T04:57:42.439 INFO:tasks.workunit.client.0.smithi152.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-04-02T04:57:42.493 INFO:tasks.workunit.client.0.smithi152.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-04-02T04:57:42.496 DEBUG:teuthology.orchestra.run.smithi152:> set -ex 2024-04-02T04:57:42.496 DEBUG:teuthology.orchestra.run.smithi152:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-04-02T04:57:42.551 INFO:tasks.workunit:Running workunits matching suites/fsstress.sh on client.0... 2024-04-02T04:57:42.553 INFO:tasks.workunit:Running workunit suites/fsstress.sh... 2024-04-02T04:57:42.553 DEBUG:teuthology.orchestra.run.smithi152:workunit test suites/fsstress.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=6cec793d9073f0594e2364b9b687ecaf22424cca TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsstress.sh 2024-04-02T04:57:42.612 INFO:tasks.workunit.client.0.smithi152.stderr:+ mkdir -p fsstress 2024-04-02T04:57:42.614 INFO:tasks.workunit.client.0.smithi152.stderr:+ pushd fsstress 2024-04-02T04:57:42.614 INFO:tasks.workunit.client.0.smithi152.stdout:~/cephtest/mnt.0/client.0/tmp/fsstress ~/cephtest/mnt.0/client.0/tmp 2024-04-02T04:57:42.615 INFO:tasks.workunit.client.0.smithi152.stderr:+ wget -q -O ltp-full.tgz http://download.ceph.com/qa/ltp-full-20091231.tgz 2024-04-02T04:57:43.480 INFO:tasks.workunit.client.0.smithi152.stderr:+ tar xzf ltp-full.tgz 2024-04-02T04:57:43.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:43 smithi154 ceph-mon[35729]: pgmap v230: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 3 op/s 2024-04-02T04:57:43.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:43 smithi152 ceph-mon[29073]: pgmap v230: 65 pgs: 65 active+clean; 589 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 1.7 KiB/s rd, 3 op/s 2024-04-02T04:57:45.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:45 smithi154 ceph-mon[35729]: pgmap v231: 65 pgs: 65 active+clean; 2.7 MiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 3.7 KiB/s rd, 179 KiB/s wr, 5 op/s 2024-04-02T04:57:45.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:45 smithi152 ceph-mon[29073]: pgmap v231: 65 pgs: 65 active+clean; 2.7 MiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 3.7 KiB/s rd, 179 KiB/s wr, 5 op/s 2024-04-02T04:57:47.483 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:47 smithi154 ceph-mon[35729]: pgmap v232: 65 pgs: 65 active+clean; 2.7 MiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.8 KiB/s rd, 179 KiB/s wr, 3 op/s 2024-04-02T04:57:47.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:47 smithi152 ceph-mon[29073]: pgmap v232: 65 pgs: 65 active+clean; 2.7 MiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 2.8 KiB/s rd, 179 KiB/s wr, 3 op/s 2024-04-02T04:57:48.643 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:48 smithi154 ceph-mon[35729]: Upgrade: Updating mgr.smithi154.pplnxz 2024-04-02T04:57:48.643 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:48 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:57:48.643 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:48 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi154.pplnxz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T04:57:48.643 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:48 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T04:57:48.643 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:48 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:57:48.643 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:48 smithi154 ceph-mon[35729]: Deploying daemon mgr.smithi154.pplnxz on smithi154 2024-04-02T04:57:48.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:48 smithi152 ceph-mon[29073]: Upgrade: Updating mgr.smithi154.pplnxz 2024-04-02T04:57:48.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:48 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:57:48.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:48 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi154.pplnxz", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-02T04:57:48.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:48 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-02T04:57:48.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:48 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-02T04:57:48.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:48 smithi152 ceph-mon[29073]: Deploying daemon mgr.smithi154.pplnxz on smithi154 2024-04-02T04:57:49.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:49 smithi154 ceph-mon[35729]: pgmap v233: 65 pgs: 65 active+clean; 7.8 MiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 344 KiB/s rd, 643 KiB/s wr, 31 op/s 2024-04-02T04:57:49.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:49 smithi152 ceph-mon[29073]: pgmap v233: 65 pgs: 65 active+clean; 7.8 MiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 344 KiB/s rd, 643 KiB/s wr, 31 op/s 2024-04-02T04:57:50.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:50 smithi154 ceph-mon[35729]: fsmap cephfs:2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi152.ywghzy=up:active} 2 up:standby-replay 2024-04-02T04:57:50.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:50 smithi152 ceph-mon[29073]: fsmap cephfs:2 {0=cephfs.smithi152.lluqrs=up:active,1=cephfs.smithi152.ywghzy=up:active} 2 up:standby-replay 2024-04-02T04:57:51.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:51 smithi154 ceph-mon[35729]: pgmap v234: 65 pgs: 65 active+clean; 31 MiB data, 1.8 GiB used, 535 GiB / 536 GiB avail; 1009 KiB/s rd, 2.7 MiB/s wr, 270 op/s 2024-04-02T04:57:51.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:51 smithi152 ceph-mon[29073]: pgmap v234: 65 pgs: 65 active+clean; 31 MiB data, 1.8 GiB used, 535 GiB / 536 GiB avail; 1009 KiB/s rd, 2.7 MiB/s wr, 270 op/s 2024-04-02T04:57:52.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:52 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:57:52.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:52 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:57:52.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:52 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:57:52.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:52 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:57:52.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:52 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:57:52.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:52 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-02T04:57:52.868 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.863+0000 7fc15954d700 1 -- 172.21.15.152:0/612524182 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc1540fe400 msgr2=0x7fc1540fe810 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:52.869 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.863+0000 7fc15954d700 1 --2- 172.21.15.152:0/612524182 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc1540fe400 0x7fc1540fe810 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 crypto rx=0x7fc14c009ce0 tx=0x7fc14c004d10 comp rx=0 tx=0).stop 2024-04-02T04:57:52.869 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.863+0000 7fc15954d700 1 -- 172.21.15.152:0/612524182 shutdown_connections 2024-04-02T04:57:52.869 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.863+0000 7fc15954d700 1 --2- 172.21.15.152:0/612524182 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc1540fed50 0x7fc1540fae90 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:52.869 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.863+0000 7fc15954d700 1 --2- 172.21.15.152:0/612524182 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc1540fe400 0x7fc1540fe810 unknown :-1 s=CLOSED pgs=302 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:52.869 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.863+0000 7fc15954d700 1 -- 172.21.15.152:0/612524182 >> 172.21.15.152:0/612524182 conn(0x7fc1540f6ba0 msgr2=0x7fc1540f8fd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:52.869 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.863+0000 7fc15954d700 1 -- 172.21.15.152:0/612524182 shutdown_connections 2024-04-02T04:57:52.869 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.863+0000 7fc15954d700 1 -- 172.21.15.152:0/612524182 wait complete. 2024-04-02T04:57:52.869 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.864+0000 7fc15954d700 1 Processor -- start 2024-04-02T04:57:52.869 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.864+0000 7fc15954d700 1 -- start start 2024-04-02T04:57:52.869 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.864+0000 7fc15954d700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc1540fe400 0x7fc1540fdbf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:52.869 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.864+0000 7fc15954d700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc1540fed50 0x7fc1540fc0c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:52.869 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.864+0000 7fc15954d700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc1540fc670 con 0x7fc1540fed50 2024-04-02T04:57:52.869 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.864+0000 7fc15954d700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc1540fc7e0 con 0x7fc1540fe400 2024-04-02T04:57:52.870 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.864+0000 7fc1527fc700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc1540fed50 0x7fc1540fc0c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:52.870 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.864+0000 7fc1527fc700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc1540fed50 0x7fc1540fc0c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:42782/0 (socket says 172.21.15.152:42782) 2024-04-02T04:57:52.870 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.864+0000 7fc1527fc700 1 -- 172.21.15.152:0/2821545826 learned_addr learned my addr 172.21.15.152:0/2821545826 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:57:52.870 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.864+0000 7fc152ffd700 1 --2- 172.21.15.152:0/2821545826 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc1540fe400 0x7fc1540fdbf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:52.870 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.865+0000 7fc152ffd700 1 -- 172.21.15.152:0/2821545826 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc1540fed50 msgr2=0x7fc1540fc0c0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:52.870 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.865+0000 7fc152ffd700 1 --2- 172.21.15.152:0/2821545826 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc1540fed50 0x7fc1540fc0c0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:52.870 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.865+0000 7fc152ffd700 1 -- 172.21.15.152:0/2821545826 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc14c009940 con 0x7fc1540fe400 2024-04-02T04:57:52.870 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.865+0000 7fc1527fc700 1 --2- 172.21.15.152:0/2821545826 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc1540fed50 0x7fc1540fc0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-02T04:57:52.870 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.865+0000 7fc152ffd700 1 --2- 172.21.15.152:0/2821545826 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc1540fe400 0x7fc1540fdbf0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7fc14c009cb0 tx=0x7fc14c004d10 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:52.871 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.866+0000 7fc14b7fe700 1 -- 172.21.15.152:0/2821545826 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fc14c00e040 con 0x7fc1540fe400 2024-04-02T04:57:52.871 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.866+0000 7fc14b7fe700 1 -- 172.21.15.152:0/2821545826 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fc14c01cd00 con 0x7fc1540fe400 2024-04-02T04:57:52.871 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.866+0000 7fc14b7fe700 1 -- 172.21.15.152:0/2821545826 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fc14c012b90 con 0x7fc1540fe400 2024-04-02T04:57:52.871 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.866+0000 7fc15954d700 1 -- 172.21.15.152:0/2821545826 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc1540fca60 con 0x7fc1540fe400 2024-04-02T04:57:52.871 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.866+0000 7fc15954d700 1 -- 172.21.15.152:0/2821545826 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc154115a20 con 0x7fc1540fe400 2024-04-02T04:57:52.871 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.867+0000 7fc15954d700 1 -- 172.21.15.152:0/2821545826 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc15405f5f0 con 0x7fc1540fe400 2024-04-02T04:57:52.872 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.868+0000 7fc14b7fe700 1 -- 172.21.15.152:0/2821545826 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7fc14c01c830 con 0x7fc1540fe400 2024-04-02T04:57:52.872 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.868+0000 7fc14b7fe700 1 --2- 172.21.15.152:0/2821545826 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fc1400692a0 0x7fc14006b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:52.872 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.868+0000 7fc14b7fe700 1 -- 172.21.15.152:0/2821545826 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7fc14c089360 con 0x7fc1540fe400 2024-04-02T04:57:52.872 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.868+0000 7fc1527fc700 1 --2- 172.21.15.152:0/2821545826 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fc1400692a0 0x7fc14006b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:52.873 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.869+0000 7fc1527fc700 1 --2- 172.21.15.152:0/2821545826 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fc1400692a0 0x7fc14006b750 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7fc13c0041d0 tx=0x7fc13c00b040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:52.876 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:52.873+0000 7fc14b7fe700 1 -- 172.21.15.152:0/2821545826 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fc14c055640 con 0x7fc1540fe400 2024-04-02T04:57:53.132 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.128+0000 7fc15954d700 1 -- 172.21.15.152:0/2821545826 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}) v1 -- 0x7fc1541024d0 con 0x7fc1400692a0 2024-04-02T04:57:53.133 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.130+0000 7fc14b7fe700 1 -- 172.21.15.152:0/2821545826 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+328 (secure 0 0 0) 0x7fc1541024d0 con 0x7fc1400692a0 2024-04-02T04:57:53.137 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.132+0000 7fc15954d700 1 -- 172.21.15.152:0/2821545826 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fc1400692a0 msgr2=0x7fc14006b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:53.137 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.132+0000 7fc15954d700 1 --2- 172.21.15.152:0/2821545826 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fc1400692a0 0x7fc14006b750 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 crypto rx=0x7fc13c0041d0 tx=0x7fc13c00b040 comp rx=0 tx=0).stop 2024-04-02T04:57:53.137 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.132+0000 7fc15954d700 1 -- 172.21.15.152:0/2821545826 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc1540fe400 msgr2=0x7fc1540fdbf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:53.137 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.132+0000 7fc15954d700 1 --2- 172.21.15.152:0/2821545826 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc1540fe400 0x7fc1540fdbf0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 crypto rx=0x7fc14c009cb0 tx=0x7fc14c004d10 comp rx=0 tx=0).stop 2024-04-02T04:57:53.137 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.132+0000 7fc15954d700 1 -- 172.21.15.152:0/2821545826 shutdown_connections 2024-04-02T04:57:53.137 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.132+0000 7fc15954d700 1 --2- 172.21.15.152:0/2821545826 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fc1400692a0 0x7fc14006b750 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:53.137 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.133+0000 7fc15954d700 1 --2- 172.21.15.152:0/2821545826 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fc1540fe400 0x7fc1540fdbf0 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:53.137 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.133+0000 7fc15954d700 1 --2- 172.21.15.152:0/2821545826 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fc1540fed50 0x7fc1540fc0c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:53.137 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.133+0000 7fc15954d700 1 -- 172.21.15.152:0/2821545826 >> 172.21.15.152:0/2821545826 conn(0x7fc1540f6ba0 msgr2=0x7fc1540f82c0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:53.137 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.133+0000 7fc15954d700 1 -- 172.21.15.152:0/2821545826 shutdown_connections 2024-04-02T04:57:53.137 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.133+0000 7fc15954d700 1 -- 172.21.15.152:0/2821545826 wait complete. 2024-04-02T04:57:53.150 INFO:teuthology.orchestra.run.smithi152.stdout:true 2024-04-02T04:57:53.277 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.273+0000 7fb1d28b5700 1 -- 172.21.15.152:0/2279465596 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb1cc0fc6f0 msgr2=0x7fb1cc0fcb60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:53.277 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.273+0000 7fb1d28b5700 1 --2- 172.21.15.152:0/2279465596 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb1cc0fc6f0 0x7fb1cc0fcb60 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 crypto rx=0x7fb1c0004940 tx=0x7fb1c0009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:53.277 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.273+0000 7fb1d28b5700 1 -- 172.21.15.152:0/2279465596 shutdown_connections 2024-04-02T04:57:53.277 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.273+0000 7fb1d28b5700 1 --2- 172.21.15.152:0/2279465596 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb1cc0fc6f0 0x7fb1cc0fcb60 unknown :-1 s=CLOSED pgs=303 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:53.278 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.273+0000 7fb1d28b5700 1 --2- 172.21.15.152:0/2279465596 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb1cc0fb440 0x7fb1cc0fb850 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:53.278 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.273+0000 7fb1d28b5700 1 -- 172.21.15.152:0/2279465596 >> 172.21.15.152:0/2279465596 conn(0x7fb1cc0f6ba0 msgr2=0x7fb1cc0f8fd0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:53.278 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.273+0000 7fb1d28b5700 1 -- 172.21.15.152:0/2279465596 shutdown_connections 2024-04-02T04:57:53.278 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.273+0000 7fb1d28b5700 1 -- 172.21.15.152:0/2279465596 wait complete. 2024-04-02T04:57:53.279 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.274+0000 7fb1d28b5700 1 Processor -- start 2024-04-02T04:57:53.279 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.274+0000 7fb1d28b5700 1 -- start start 2024-04-02T04:57:53.279 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.274+0000 7fb1d28b5700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb1cc0fb440 0x7fb1cc110670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:53.279 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.274+0000 7fb1d28b5700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb1cc0fc6f0 0x7fb1cc112bc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:53.279 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.274+0000 7fb1d28b5700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb1cc1026e0 con 0x7fb1cc0fb440 2024-04-02T04:57:53.279 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.274+0000 7fb1d28b5700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb1cc102820 con 0x7fb1cc0fc6f0 2024-04-02T04:57:53.280 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.274+0000 7fb1cbfff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb1cc0fb440 0x7fb1cc110670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:53.280 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.274+0000 7fb1cbfff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb1cc0fb440 0x7fb1cc110670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:42800/0 (socket says 172.21.15.152:42800) 2024-04-02T04:57:53.280 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.274+0000 7fb1cbfff700 1 -- 172.21.15.152:0/1332415045 learned_addr learned my addr 172.21.15.152:0/1332415045 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:57:53.280 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.274+0000 7fb1cb7fe700 1 --2- 172.21.15.152:0/1332415045 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb1cc0fc6f0 0x7fb1cc112bc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:53.280 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.275+0000 7fb1cbfff700 1 -- 172.21.15.152:0/1332415045 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb1cc0fc6f0 msgr2=0x7fb1cc112bc0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:53.280 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.275+0000 7fb1cbfff700 1 --2- 172.21.15.152:0/1332415045 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb1cc0fc6f0 0x7fb1cc112bc0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:53.280 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.275+0000 7fb1cbfff700 1 -- 172.21.15.152:0/1332415045 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb1c0005040 con 0x7fb1cc0fb440 2024-04-02T04:57:53.280 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.275+0000 7fb1cb7fe700 1 --2- 172.21.15.152:0/1332415045 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb1cc0fc6f0 0x7fb1cc112bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-02T04:57:53.280 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.275+0000 7fb1cbfff700 1 --2- 172.21.15.152:0/1332415045 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb1cc0fb440 0x7fb1cc110670 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7fb1bc00eb40 tx=0x7fb1bc00b040 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:53.280 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.275+0000 7fb1c8ff9700 1 -- 172.21.15.152:0/1332415045 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fb1bc007cc0 con 0x7fb1cc0fb440 2024-04-02T04:57:53.280 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.275+0000 7fb1d28b5700 1 -- 172.21.15.152:0/1332415045 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb1cc113160 con 0x7fb1cc0fb440 2024-04-02T04:57:53.280 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.275+0000 7fb1c8ff9700 1 -- 172.21.15.152:0/1332415045 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fb1bc00f040 con 0x7fb1cc0fb440 2024-04-02T04:57:53.281 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.275+0000 7fb1d28b5700 1 -- 172.21.15.152:0/1332415045 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb1cc1136a0 con 0x7fb1cc0fb440 2024-04-02T04:57:53.281 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.275+0000 7fb1c8ff9700 1 -- 172.21.15.152:0/1332415045 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fb1bc00c9b0 con 0x7fb1cc0fb440 2024-04-02T04:57:53.281 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.276+0000 7fb1d28b5700 1 -- 172.21.15.152:0/1332415045 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb1cc05f5f0 con 0x7fb1cc0fb440 2024-04-02T04:57:53.281 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.277+0000 7fb1c8ff9700 1 -- 172.21.15.152:0/1332415045 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7fb1bc00cb10 con 0x7fb1cc0fb440 2024-04-02T04:57:53.282 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.278+0000 7fb1c8ff9700 1 --2- 172.21.15.152:0/1332415045 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb1b40692a0 0x7fb1b406b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:53.282 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.278+0000 7fb1c8ff9700 1 -- 172.21.15.152:0/1332415045 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7fb1bc086dc0 con 0x7fb1cc0fb440 2024-04-02T04:57:53.282 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.278+0000 7fb1cb7fe700 1 --2- 172.21.15.152:0/1332415045 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb1b40692a0 0x7fb1b406b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:53.282 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.278+0000 7fb1cb7fe700 1 --2- 172.21.15.152:0/1332415045 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb1b40692a0 0x7fb1b406b750 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7fb1c000b0d0 tx=0x7fb1c0009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:53.288 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.284+0000 7fb1c8ff9700 1 -- 172.21.15.152:0/1332415045 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fb1bc04e190 con 0x7fb1cc0fb440 2024-04-02T04:57:53.577 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.573+0000 7fb1d28b5700 1 -- 172.21.15.152:0/1332415045 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}) v1 -- 0x7fb1cc100990 con 0x7fb1b40692a0 2024-04-02T04:57:53.577 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.574+0000 7fb1c8ff9700 1 -- 172.21.15.152:0/1332415045 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+328 (secure 0 0 0) 0x7fb1cc100990 con 0x7fb1b40692a0 2024-04-02T04:57:53.582 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.577+0000 7fb1d28b5700 1 -- 172.21.15.152:0/1332415045 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb1b40692a0 msgr2=0x7fb1b406b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:53.582 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.577+0000 7fb1d28b5700 1 --2- 172.21.15.152:0/1332415045 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb1b40692a0 0x7fb1b406b750 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 crypto rx=0x7fb1c000b0d0 tx=0x7fb1c0009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:53.582 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.577+0000 7fb1d28b5700 1 -- 172.21.15.152:0/1332415045 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb1cc0fb440 msgr2=0x7fb1cc110670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:53.582 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.577+0000 7fb1d28b5700 1 --2- 172.21.15.152:0/1332415045 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb1cc0fb440 0x7fb1cc110670 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 crypto rx=0x7fb1bc00eb40 tx=0x7fb1bc00b040 comp rx=0 tx=0).stop 2024-04-02T04:57:53.582 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.577+0000 7fb1d28b5700 1 -- 172.21.15.152:0/1332415045 shutdown_connections 2024-04-02T04:57:53.582 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.577+0000 7fb1d28b5700 1 --2- 172.21.15.152:0/1332415045 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb1b40692a0 0x7fb1b406b750 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:53.582 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.577+0000 7fb1d28b5700 1 --2- 172.21.15.152:0/1332415045 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb1cc0fb440 0x7fb1cc110670 unknown :-1 s=CLOSED pgs=304 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:53.582 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.577+0000 7fb1d28b5700 1 --2- 172.21.15.152:0/1332415045 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb1cc0fc6f0 0x7fb1cc112bc0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:53.582 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.577+0000 7fb1d28b5700 1 -- 172.21.15.152:0/1332415045 >> 172.21.15.152:0/1332415045 conn(0x7fb1cc0f6ba0 msgr2=0x7fb1cc0f8220 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:53.582 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.578+0000 7fb1d28b5700 1 -- 172.21.15.152:0/1332415045 shutdown_connections 2024-04-02T04:57:53.582 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.578+0000 7fb1d28b5700 1 -- 172.21.15.152:0/1332415045 wait complete. 2024-04-02T04:57:53.693 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.689+0000 7fbf2c36a700 1 -- 172.21.15.152:0/4059086929 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbf24102360 msgr2=0x7fbf24104740 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:53.693 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.689+0000 7fbf2c36a700 1 --2- 172.21.15.152:0/4059086929 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbf24102360 0x7fbf24104740 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 crypto rx=0x7fbf14004940 tx=0x7fbf14009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:53.694 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.689+0000 7fbf2c36a700 1 -- 172.21.15.152:0/4059086929 shutdown_connections 2024-04-02T04:57:53.694 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.689+0000 7fbf2c36a700 1 --2- 172.21.15.152:0/4059086929 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbf24102360 0x7fbf24104740 unknown :-1 s=CLOSED pgs=305 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:53.694 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.689+0000 7fbf2c36a700 1 --2- 172.21.15.152:0/4059086929 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbf24066280 0x7fbf24101e20 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:53.694 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.689+0000 7fbf2c36a700 1 -- 172.21.15.152:0/4059086929 >> 172.21.15.152:0/4059086929 conn(0x7fbf240f7430 msgr2=0x7fbf240f9860 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:53.694 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.689+0000 7fbf2c36a700 1 -- 172.21.15.152:0/4059086929 shutdown_connections 2024-04-02T04:57:53.694 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.689+0000 7fbf2c36a700 1 -- 172.21.15.152:0/4059086929 wait complete. 2024-04-02T04:57:53.696 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.690+0000 7fbf2c36a700 1 Processor -- start 2024-04-02T04:57:53.696 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.690+0000 7fbf2c36a700 1 -- start start 2024-04-02T04:57:53.696 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.690+0000 7fbf2c36a700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbf24066280 0x7fbf2410fbb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:53.696 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.690+0000 7fbf2c36a700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbf24102360 0x7fbf241100f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:53.696 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.690+0000 7fbf2c36a700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbf241106f0 con 0x7fbf24102360 2024-04-02T04:57:53.696 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.690+0000 7fbf2c36a700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbf24110f40 con 0x7fbf24066280 2024-04-02T04:57:53.696 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.690+0000 7fbf2a106700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbf24066280 0x7fbf2410fbb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:53.696 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.690+0000 7fbf29905700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbf24102360 0x7fbf241100f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:53.696 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.690+0000 7fbf2a106700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbf24066280 0x7fbf2410fbb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.152:55462/0 (socket says 172.21.15.152:55462) 2024-04-02T04:57:53.696 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.690+0000 7fbf2a106700 1 -- 172.21.15.152:0/3302565712 learned_addr learned my addr 172.21.15.152:0/3302565712 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:57:53.696 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.691+0000 7fbf29905700 1 -- 172.21.15.152:0/3302565712 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbf24066280 msgr2=0x7fbf2410fbb0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:53.697 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.691+0000 7fbf29905700 1 --2- 172.21.15.152:0/3302565712 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbf24066280 0x7fbf2410fbb0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:53.697 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.691+0000 7fbf29905700 1 -- 172.21.15.152:0/3302565712 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbf14005040 con 0x7fbf24102360 2024-04-02T04:57:53.697 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.691+0000 7fbf2a106700 1 --2- 172.21.15.152:0/3302565712 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbf24066280 0x7fbf2410fbb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-02T04:57:53.697 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.691+0000 7fbf29905700 1 --2- 172.21.15.152:0/3302565712 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbf24102360 0x7fbf241100f0 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7fbf14000c00 tx=0x7fbf14009e30 comp rx=0 tx=0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:53.697 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.691+0000 7fbf1affd700 1 -- 172.21.15.152:0/3302565712 <== mon.0 v2:172.21.15.152:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fbf1400e070 con 0x7fbf24102360 2024-04-02T04:57:53.697 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.691+0000 7fbf1affd700 1 -- 172.21.15.152:0/3302565712 <== mon.0 v2:172.21.15.152:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fbf1401c400 con 0x7fbf24102360 2024-04-02T04:57:53.697 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.691+0000 7fbf2c36a700 1 -- 172.21.15.152:0/3302565712 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbf241111c0 con 0x7fbf24102360 2024-04-02T04:57:53.697 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.691+0000 7fbf1affd700 1 -- 172.21.15.152:0/3302565712 <== mon.0 v2:172.21.15.152:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fbf1400b650 con 0x7fbf24102360 2024-04-02T04:57:53.697 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.691+0000 7fbf2c36a700 1 -- 172.21.15.152:0/3302565712 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbf241116a0 con 0x7fbf24102360 2024-04-02T04:57:53.698 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.692+0000 7fbf2c36a700 1 -- 172.21.15.152:0/3302565712 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbf2405f5f0 con 0x7fbf24102360 2024-04-02T04:57:53.698 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.694+0000 7fbf1affd700 1 -- 172.21.15.152:0/3302565712 <== mon.0 v2:172.21.15.152:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7fbf1401ca20 con 0x7fbf24102360 2024-04-02T04:57:53.698 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.694+0000 7fbf1affd700 1 --2- 172.21.15.152:0/3302565712 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fbf100692a0 0x7fbf1006b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:53.698 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.694+0000 7fbf1affd700 1 -- 172.21.15.152:0/3302565712 <== mon.0 v2:172.21.15.152:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7fbf14088cf0 con 0x7fbf24102360 2024-04-02T04:57:53.698 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.694+0000 7fbf2a106700 1 --2- 172.21.15.152:0/3302565712 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fbf100692a0 0x7fbf1006b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:53.699 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.695+0000 7fbf2a106700 1 --2- 172.21.15.152:0/3302565712 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fbf100692a0 0x7fbf1006b750 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7fbf2000f6a0 tx=0x7fbf2000b040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:53.704 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.700+0000 7fbf1affd700 1 -- 172.21.15.152:0/3302565712 <== mon.0 v2:172.21.15.152:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fbf14058b00 con 0x7fbf24102360 2024-04-02T04:57:53.709 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:53 smithi154 ceph-mon[35729]: pgmap v235: 65 pgs: 65 active+clean; 41 MiB data, 1.8 GiB used, 535 GiB / 536 GiB avail; 1008 KiB/s rd, 3.6 MiB/s wr, 349 op/s 2024-04-02T04:57:53.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:53 smithi152 ceph-mon[29073]: pgmap v235: 65 pgs: 65 active+clean; 41 MiB data, 1.8 GiB used, 535 GiB / 536 GiB avail; 1008 KiB/s rd, 3.6 MiB/s wr, 349 op/s 2024-04-02T04:57:53.963 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.959+0000 7fbf2c36a700 1 -- 172.21.15.152:0/3302565712 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "orch ps", "target": ["mon-mgr", ""]}) v1 -- 0x7fbf24111a90 con 0x7fbf100692a0 2024-04-02T04:57:53.974 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.970+0000 7fbf1affd700 1 -- 172.21.15.152:0/3302565712 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+3410 (secure 0 0 0) 0x7fbf24111a90 con 0x7fbf100692a0 2024-04-02T04:57:53.974 INFO:teuthology.orchestra.run.smithi152.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-02T04:57:53.975 INFO:teuthology.orchestra.run.smithi152.stdout:alertmanager.smithi152 smithi152 *:9093,9094 running (6m) 94s ago 7m 25.2M - 0.25.0 c8568f914cd2 1f52e384342b 2024-04-02T04:57:53.975 INFO:teuthology.orchestra.run.smithi152.stdout:crash.smithi152 smithi152 running (7m) 94s ago 7m 6665k - 17.2.7-803-gc924f991 4948510fac0f 12f8a8489975 2024-04-02T04:57:53.975 INFO:teuthology.orchestra.run.smithi152.stdout:crash.smithi154 smithi154 running (6m) 100s ago 6m 6665k - 17.2.7-803-gc924f991 4948510fac0f 7fe215d49e0f 2024-04-02T04:57:53.975 INFO:teuthology.orchestra.run.smithi152.stdout:grafana.smithi152 smithi152 *:3000 running (5m) 94s ago 7m 85.7M - 9.4.7 954c08fa6188 eb6036b14f99 2024-04-02T04:57:53.975 INFO:teuthology.orchestra.run.smithi152.stdout:mds.cephfs.smithi152.lluqrs smithi152 running (2m) 94s ago 119s 15.0M - 17.2.7-803-gc924f991 4948510fac0f bada96a57e65 2024-04-02T04:57:53.975 INFO:teuthology.orchestra.run.smithi152.stdout:mds.cephfs.smithi152.ywghzy smithi152 running (114s) 94s ago 113s 15.9M - 17.2.7-803-gc924f991 4948510fac0f 267b6d51cad3 2024-04-02T04:57:53.975 INFO:teuthology.orchestra.run.smithi152.stdout:mds.cephfs.smithi154.llvaoh smithi154 running (117s) 100s ago 117s 11.3M - 17.2.7-803-gc924f991 4948510fac0f 82f2bed61253 2024-04-02T04:57:53.975 INFO:teuthology.orchestra.run.smithi152.stdout:mds.cephfs.smithi154.rbkmwp smithi154 running (111s) 100s ago 111s 14.7M - 17.2.7-803-gc924f991 4948510fac0f eba3a7f41a09 2024-04-02T04:57:53.975 INFO:teuthology.orchestra.run.smithi152.stdout:mgr.smithi152.jivdcl smithi152 *:9283 running (8m) 94s ago 8m 481M - 17.2.7-803-gc924f991 4948510fac0f 2fd5fbea3f34 2024-04-02T04:57:53.975 INFO:teuthology.orchestra.run.smithi152.stdout:mgr.smithi154.pplnxz smithi154 *:8443,9283 starting - - - - 2024-04-02T04:57:53.975 INFO:teuthology.orchestra.run.smithi152.stdout:mon.smithi152 smithi152 running (8m) 94s ago 8m 46.3M 2048M 17.2.7-803-gc924f991 4948510fac0f 399b9cc778f0 2024-04-02T04:57:53.975 INFO:teuthology.orchestra.run.smithi152.stdout:mon.smithi154 smithi154 running (6m) 100s ago 6m 36.0M 2048M 17.2.7-803-gc924f991 4948510fac0f 975aefa2215f 2024-04-02T04:57:53.975 INFO:teuthology.orchestra.run.smithi152.stdout:node-exporter.smithi152 smithi152 *:9100 running (7m) 94s ago 7m 16.5M - 1.5.0 0da6a335fe13 8168d9a53e30 2024-04-02T04:57:53.975 INFO:teuthology.orchestra.run.smithi152.stdout:node-exporter.smithi154 smithi154 *:9100 running (6m) 100s ago 6m 15.7M - 1.5.0 0da6a335fe13 8ee2f7610052 2024-04-02T04:57:53.975 INFO:teuthology.orchestra.run.smithi152.stdout:osd.0 smithi152 running (5m) 94s ago 5m 45.0M 4288M 17.2.7-803-gc924f991 4948510fac0f 9935e01f3199 2024-04-02T04:57:53.976 INFO:teuthology.orchestra.run.smithi152.stdout:osd.1 smithi152 running (5m) 94s ago 5m 46.5M 4288M 17.2.7-803-gc924f991 4948510fac0f e5c49d238944 2024-04-02T04:57:53.976 INFO:teuthology.orchestra.run.smithi152.stdout:osd.2 smithi152 running (4m) 94s ago 4m 47.1M 4288M 17.2.7-803-gc924f991 4948510fac0f c53a93ae7737 2024-04-02T04:57:53.976 INFO:teuthology.orchestra.run.smithi152.stdout:osd.3 smithi154 running (4m) 100s ago 4m 44.8M 5312M 17.2.7-803-gc924f991 4948510fac0f 959ebaab3713 2024-04-02T04:57:53.976 INFO:teuthology.orchestra.run.smithi152.stdout:osd.4 smithi154 running (3m) 100s ago 3m 41.7M 5312M 17.2.7-803-gc924f991 4948510fac0f 4856c813e7fd 2024-04-02T04:57:53.976 INFO:teuthology.orchestra.run.smithi152.stdout:osd.5 smithi154 running (3m) 100s ago 3m 42.0M 5312M 17.2.7-803-gc924f991 4948510fac0f fbc6bb91d1a7 2024-04-02T04:57:53.976 INFO:teuthology.orchestra.run.smithi152.stdout:prometheus.smithi152 smithi152 *:9095 running (5m) 94s ago 6m 53.8M - 2.43.0 a07b618ecd1d 874955f0a102 2024-04-02T04:57:53.977 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.973+0000 7fbf2c36a700 1 -- 172.21.15.152:0/3302565712 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fbf100692a0 msgr2=0x7fbf1006b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:53.977 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.973+0000 7fbf2c36a700 1 --2- 172.21.15.152:0/3302565712 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fbf100692a0 0x7fbf1006b750 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 crypto rx=0x7fbf2000f6a0 tx=0x7fbf2000b040 comp rx=0 tx=0).stop 2024-04-02T04:57:53.977 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.973+0000 7fbf2c36a700 1 -- 172.21.15.152:0/3302565712 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbf24102360 msgr2=0x7fbf241100f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:53.978 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.973+0000 7fbf2c36a700 1 --2- 172.21.15.152:0/3302565712 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbf24102360 0x7fbf241100f0 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 crypto rx=0x7fbf14000c00 tx=0x7fbf14009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:53.978 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.973+0000 7fbf2c36a700 1 -- 172.21.15.152:0/3302565712 shutdown_connections 2024-04-02T04:57:53.978 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.973+0000 7fbf2c36a700 1 --2- 172.21.15.152:0/3302565712 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fbf100692a0 0x7fbf1006b750 unknown :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:53.978 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.973+0000 7fbf2c36a700 1 --2- 172.21.15.152:0/3302565712 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fbf24066280 0x7fbf2410fbb0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:53.978 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.973+0000 7fbf2c36a700 1 --2- 172.21.15.152:0/3302565712 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fbf24102360 0x7fbf241100f0 unknown :-1 s=CLOSED pgs=306 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:53.978 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.973+0000 7fbf2c36a700 1 -- 172.21.15.152:0/3302565712 >> 172.21.15.152:0/3302565712 conn(0x7fbf240f7430 msgr2=0x7fbf240f7850 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:53.978 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.973+0000 7fbf2c36a700 1 -- 172.21.15.152:0/3302565712 shutdown_connections 2024-04-02T04:57:53.978 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:53.973+0000 7fbf2c36a700 1 -- 172.21.15.152:0/3302565712 wait complete. 2024-04-02T04:57:54.093 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.088+0000 7fb4841cc700 1 -- 172.21.15.152:0/2613216009 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb47c0fcee0 msgr2=0x7fb47c0fd310 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:54.094 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.088+0000 7fb4841cc700 1 --2- 172.21.15.152:0/2613216009 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb47c0fcee0 0x7fb47c0fd310 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 crypto rx=0x7fb478004940 tx=0x7fb478009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:54.094 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.088+0000 7fb4841cc700 1 -- 172.21.15.152:0/2613216009 shutdown_connections 2024-04-02T04:57:54.094 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.088+0000 7fb4841cc700 1 --2- 172.21.15.152:0/2613216009 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb47c0fe0d0 0x7fb47c0fe540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:54.094 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.088+0000 7fb4841cc700 1 --2- 172.21.15.152:0/2613216009 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb47c0fcee0 0x7fb47c0fd310 unknown :-1 s=CLOSED pgs=307 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:54.094 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.088+0000 7fb4841cc700 1 -- 172.21.15.152:0/2613216009 >> 172.21.15.152:0/2613216009 conn(0x7fb47c0f8560 msgr2=0x7fb47c0fa970 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:54.094 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.089+0000 7fb4841cc700 1 -- 172.21.15.152:0/2613216009 shutdown_connections 2024-04-02T04:57:54.094 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.089+0000 7fb4841cc700 1 -- 172.21.15.152:0/2613216009 wait complete. 2024-04-02T04:57:54.094 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.089+0000 7fb4841cc700 1 Processor -- start 2024-04-02T04:57:54.094 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.089+0000 7fb4841cc700 1 -- start start 2024-04-02T04:57:54.094 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.089+0000 7fb4841cc700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb47c0fcee0 0x7fb47c10d9d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:54.094 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.089+0000 7fb4841cc700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb47c0fe0d0 0x7fb47c10df10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:54.094 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.089+0000 7fb4841cc700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb47c10e510 con 0x7fb47c0fe0d0 2024-04-02T04:57:54.095 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.089+0000 7fb4841cc700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb47c10ed60 con 0x7fb47c0fcee0 2024-04-02T04:57:54.095 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.089+0000 7fb481767700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb47c0fe0d0 0x7fb47c10df10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:54.095 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.090+0000 7fb481f68700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb47c0fcee0 0x7fb47c10d9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:54.095 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.090+0000 7fb481f68700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb47c0fcee0 0x7fb47c10d9d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.152:55488/0 (socket says 172.21.15.152:55488) 2024-04-02T04:57:54.095 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.090+0000 7fb481f68700 1 -- 172.21.15.152:0/1862274938 learned_addr learned my addr 172.21.15.152:0/1862274938 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:57:54.095 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.090+0000 7fb481f68700 1 -- 172.21.15.152:0/1862274938 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb47c0fe0d0 msgr2=0x7fb47c10df10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:54.095 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.090+0000 7fb481f68700 1 --2- 172.21.15.152:0/1862274938 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb47c0fe0d0 0x7fb47c10df10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:54.095 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.090+0000 7fb481f68700 1 -- 172.21.15.152:0/1862274938 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb478005040 con 0x7fb47c0fcee0 2024-04-02T04:57:54.097 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.090+0000 7fb481f68700 1 --2- 172.21.15.152:0/1862274938 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb47c0fcee0 0x7fb47c10d9d0 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7fb4780040d0 tx=0x7fb478009e30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:54.097 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.090+0000 7fb4727fc700 1 -- 172.21.15.152:0/1862274938 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fb47800e070 con 0x7fb47c0fcee0 2024-04-02T04:57:54.097 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.090+0000 7fb4841cc700 1 -- 172.21.15.152:0/1862274938 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb47c10efe0 con 0x7fb47c0fcee0 2024-04-02T04:57:54.097 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.090+0000 7fb4727fc700 1 -- 172.21.15.152:0/1862274938 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7fb47801c400 con 0x7fb47c0fcee0 2024-04-02T04:57:54.097 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.090+0000 7fb4727fc700 1 -- 172.21.15.152:0/1862274938 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fb47800b610 con 0x7fb47c0fcee0 2024-04-02T04:57:54.097 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.090+0000 7fb4841cc700 1 -- 172.21.15.152:0/1862274938 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb47c10f4c0 con 0x7fb47c0fcee0 2024-04-02T04:57:54.097 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.091+0000 7fb4841cc700 1 -- 172.21.15.152:0/1862274938 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb47c05f5f0 con 0x7fb47c0fcee0 2024-04-02T04:57:54.097 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.092+0000 7fb4727fc700 1 -- 172.21.15.152:0/1862274938 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7fb47801ca50 con 0x7fb47c0fcee0 2024-04-02T04:57:54.097 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.092+0000 7fb4727fc700 1 --2- 172.21.15.152:0/1862274938 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb4680692b0 0x7fb46806b760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:54.097 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.092+0000 7fb4727fc700 1 -- 172.21.15.152:0/1862274938 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7fb4780888d0 con 0x7fb47c0fcee0 2024-04-02T04:57:54.097 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.092+0000 7fb481767700 1 --2- 172.21.15.152:0/1862274938 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb4680692b0 0x7fb46806b760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:54.098 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.093+0000 7fb481767700 1 --2- 172.21.15.152:0/1862274938 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb4680692b0 0x7fb46806b760 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7fb46c009b80 tx=0x7fb46c00b040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:54.100 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.096+0000 7fb4727fc700 1 -- 172.21.15.152:0/1862274938 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7fb4780585a0 con 0x7fb47c0fcee0 2024-04-02T04:57:54.422 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.418+0000 7fb4841cc700 1 -- 172.21.15.152:0/1862274938 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "versions"} v 0) v1 -- 0x7fb47c10f770 con 0x7fb47c0fcee0 2024-04-02T04:57:54.423 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.419+0000 7fb4727fc700 1 -- 172.21.15.152:0/1862274938 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "versions"}]=0 v0) v1 ==== 56+0+638 (secure 0 0 0) 0x7fb478058130 con 0x7fb47c0fcee0 2024-04-02T04:57:54.423 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-04-02T04:57:54.423 INFO:teuthology.orchestra.run.smithi152.stdout: "mon": { 2024-04-02T04:57:54.423 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 17.2.7-803-gc924f991 (c924f9914f2809997b363f77c5cbb14929881220) quincy (stable)": 2 2024-04-02T04:57:54.423 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-04-02T04:57:54.423 INFO:teuthology.orchestra.run.smithi152.stdout: "mgr": { 2024-04-02T04:57:54.423 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 17.2.7-803-gc924f991 (c924f9914f2809997b363f77c5cbb14929881220) quincy (stable)": 2 2024-04-02T04:57:54.424 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-04-02T04:57:54.424 INFO:teuthology.orchestra.run.smithi152.stdout: "osd": { 2024-04-02T04:57:54.424 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 17.2.7-803-gc924f991 (c924f9914f2809997b363f77c5cbb14929881220) quincy (stable)": 6 2024-04-02T04:57:54.424 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-04-02T04:57:54.424 INFO:teuthology.orchestra.run.smithi152.stdout: "mds": { 2024-04-02T04:57:54.424 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 17.2.7-803-gc924f991 (c924f9914f2809997b363f77c5cbb14929881220) quincy (stable)": 4 2024-04-02T04:57:54.424 INFO:teuthology.orchestra.run.smithi152.stdout: }, 2024-04-02T04:57:54.424 INFO:teuthology.orchestra.run.smithi152.stdout: "overall": { 2024-04-02T04:57:54.424 INFO:teuthology.orchestra.run.smithi152.stdout: "ceph version 17.2.7-803-gc924f991 (c924f9914f2809997b363f77c5cbb14929881220) quincy (stable)": 14 2024-04-02T04:57:54.424 INFO:teuthology.orchestra.run.smithi152.stdout: } 2024-04-02T04:57:54.424 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-04-02T04:57:54.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.421+0000 7fb4841cc700 1 -- 172.21.15.152:0/1862274938 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb4680692b0 msgr2=0x7fb46806b760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:54.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.421+0000 7fb4841cc700 1 --2- 172.21.15.152:0/1862274938 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb4680692b0 0x7fb46806b760 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 crypto rx=0x7fb46c009b80 tx=0x7fb46c00b040 comp rx=0 tx=0).stop 2024-04-02T04:57:54.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.421+0000 7fb4841cc700 1 -- 172.21.15.152:0/1862274938 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb47c0fcee0 msgr2=0x7fb47c10d9d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:54.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.421+0000 7fb4841cc700 1 --2- 172.21.15.152:0/1862274938 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb47c0fcee0 0x7fb47c10d9d0 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 crypto rx=0x7fb4780040d0 tx=0x7fb478009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:54.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.421+0000 7fb4841cc700 1 -- 172.21.15.152:0/1862274938 shutdown_connections 2024-04-02T04:57:54.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.421+0000 7fb4841cc700 1 --2- 172.21.15.152:0/1862274938 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7fb4680692b0 0x7fb46806b760 unknown :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:54.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.421+0000 7fb4841cc700 1 --2- 172.21.15.152:0/1862274938 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7fb47c0fcee0 0x7fb47c10d9d0 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:54.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.421+0000 7fb4841cc700 1 --2- 172.21.15.152:0/1862274938 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7fb47c0fe0d0 0x7fb47c10df10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:54.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.421+0000 7fb4841cc700 1 -- 172.21.15.152:0/1862274938 >> 172.21.15.152:0/1862274938 conn(0x7fb47c0f8560 msgr2=0x7fb47c0f9c10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:54.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.421+0000 7fb4841cc700 1 -- 172.21.15.152:0/1862274938 shutdown_connections 2024-04-02T04:57:54.426 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.421+0000 7fb4841cc700 1 -- 172.21.15.152:0/1862274938 wait complete. 2024-04-02T04:57:54.544 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.539+0000 7f085a920700 1 -- 172.21.15.152:0/2855257248 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f08540fbce0 msgr2=0x7f08540fc0f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:54.544 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.539+0000 7f085a920700 1 --2- 172.21.15.152:0/2855257248 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f08540fbce0 0x7f08540fc0f0 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 crypto rx=0x7f0844004940 tx=0x7f0844009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:54.544 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.540+0000 7f085a920700 1 -- 172.21.15.152:0/2855257248 shutdown_connections 2024-04-02T04:57:54.544 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.540+0000 7f085a920700 1 --2- 172.21.15.152:0/2855257248 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f08540fcf90 0x7f08540fd400 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:54.544 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.540+0000 7f085a920700 1 --2- 172.21.15.152:0/2855257248 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f08540fbce0 0x7f08540fc0f0 unknown :-1 s=CLOSED pgs=308 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:54.544 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.540+0000 7f085a920700 1 -- 172.21.15.152:0/2855257248 >> 172.21.15.152:0/2855257248 conn(0x7f08540f7440 msgr2=0x7f08540f9870 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:54.544 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.540+0000 7f085a920700 1 -- 172.21.15.152:0/2855257248 shutdown_connections 2024-04-02T04:57:54.544 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.540+0000 7f085a920700 1 -- 172.21.15.152:0/2855257248 wait complete. 2024-04-02T04:57:54.545 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.540+0000 7f085a920700 1 Processor -- start 2024-04-02T04:57:54.545 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.541+0000 7f085a920700 1 -- start start 2024-04-02T04:57:54.545 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.541+0000 7f085a920700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f08540fbce0 0x7f0854110700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:54.545 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.541+0000 7f085a920700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f08540fcf90 0x7f0854112c50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:54.546 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.541+0000 7f085a920700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0854066be0 con 0x7f08540fcf90 2024-04-02T04:57:54.546 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.541+0000 7f085a920700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0854066d20 con 0x7f08540fbce0 2024-04-02T04:57:54.546 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.541+0000 7f08537fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f08540fcf90 0x7f0854112c50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:54.546 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.541+0000 7f0853fff700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f08540fbce0 0x7f0854110700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:54.546 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.541+0000 7f08537fe700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f08540fcf90 0x7f0854112c50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:42840/0 (socket says 172.21.15.152:42840) 2024-04-02T04:57:54.546 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.541+0000 7f0853fff700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f08540fbce0 0x7f0854110700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.154:3300/0 says I am v2:172.21.15.152:55506/0 (socket says 172.21.15.152:55506) 2024-04-02T04:57:54.546 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.541+0000 7f08537fe700 1 -- 172.21.15.152:0/1198141098 learned_addr learned my addr 172.21.15.152:0/1198141098 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:57:54.546 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.541+0000 7f0853fff700 1 -- 172.21.15.152:0/1198141098 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f08540fcf90 msgr2=0x7f0854112c50 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:54.546 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.541+0000 7f0853fff700 1 --2- 172.21.15.152:0/1198141098 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f08540fcf90 0x7f0854112c50 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:54.546 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.541+0000 7f0853fff700 1 -- 172.21.15.152:0/1198141098 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0844005040 con 0x7f08540fbce0 2024-04-02T04:57:54.546 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.542+0000 7f0853fff700 1 --2- 172.21.15.152:0/1198141098 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f08540fbce0 0x7f0854110700 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f084400b0d0 tx=0x7f0844009e30 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:54.546 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.542+0000 7f0850ff9700 1 -- 172.21.15.152:0/1198141098 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f084400e070 con 0x7f08540fbce0 2024-04-02T04:57:54.547 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.542+0000 7f085a920700 1 -- 172.21.15.152:0/1198141098 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f085410c4e0 con 0x7f08540fbce0 2024-04-02T04:57:54.547 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.542+0000 7f085a920700 1 -- 172.21.15.152:0/1198141098 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f085410c9f0 con 0x7f08540fbce0 2024-04-02T04:57:54.547 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.542+0000 7f0850ff9700 1 -- 172.21.15.152:0/1198141098 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f084401c400 con 0x7f08540fbce0 2024-04-02T04:57:54.547 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.542+0000 7f0850ff9700 1 -- 172.21.15.152:0/1198141098 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f084400b650 con 0x7f08540fbce0 2024-04-02T04:57:54.547 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.543+0000 7f085a920700 1 -- 172.21.15.152:0/1198141098 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f08400052e0 con 0x7f08540fbce0 2024-04-02T04:57:54.548 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.544+0000 7f0850ff9700 1 -- 172.21.15.152:0/1198141098 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f0844013af0 con 0x7f08540fbce0 2024-04-02T04:57:54.548 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.544+0000 7f0850ff9700 1 --2- 172.21.15.152:0/1198141098 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f083c069300 0x7f083c06b7b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:54.548 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.544+0000 7f0850ff9700 1 -- 172.21.15.152:0/1198141098 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7f0844088330 con 0x7f08540fbce0 2024-04-02T04:57:54.548 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.544+0000 7f08537fe700 1 --2- 172.21.15.152:0/1198141098 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f083c069300 0x7f083c06b7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:54.548 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.545+0000 7f08537fe700 1 --2- 172.21.15.152:0/1198141098 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f083c069300 0x7f083c06b7b0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f0848008fd0 tx=0x7f084800b040 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:54.552 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.549+0000 7f0850ff9700 1 -- 172.21.15.152:0/1198141098 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f08440546b0 con 0x7f08540fbce0 2024-04-02T04:57:54.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:54 smithi154 ceph-mon[35729]: from='client.24397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:57:54.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:54 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/1862274938' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T04:57:54.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:54 smithi152 ceph-mon[29073]: from='client.24397 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:57:54.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:54 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/1862274938' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-02T04:57:54.837 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.834+0000 7f085a920700 1 -- 172.21.15.152:0/1198141098 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "fs dump"} v 0) v1 -- 0x7f0840005da0 con 0x7f08540fbce0 2024-04-02T04:57:54.838 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.834+0000 7f0850ff9700 1 -- 172.21.15.152:0/1198141098 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "fs dump"}]=0 dumped fsmap epoch 21 v21) v1 ==== 76+0+1850 (secure 0 0 0) 0x7f0844004490 con 0x7f08540fbce0 2024-04-02T04:57:54.838 INFO:teuthology.orchestra.run.smithi152.stdout:e21 2024-04-02T04:57:54.838 INFO:teuthology.orchestra.run.smithi152.stdout:enable_multiple, ever_enabled_multiple: 1,1 2024-04-02T04:57:54.838 INFO:teuthology.orchestra.run.smithi152.stdout:default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-02T04:57:54.838 INFO:teuthology.orchestra.run.smithi152.stdout:legacy client fscid: 1 2024-04-02T04:57:54.839 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:57:54.839 INFO:teuthology.orchestra.run.smithi152.stdout:Filesystem 'cephfs' (1) 2024-04-02T04:57:54.839 INFO:teuthology.orchestra.run.smithi152.stdout:fs_name cephfs 2024-04-02T04:57:54.839 INFO:teuthology.orchestra.run.smithi152.stdout:epoch 21 2024-04-02T04:57:54.839 INFO:teuthology.orchestra.run.smithi152.stdout:flags 32 joinable allow_snaps allow_multimds_snaps allow_standby_replay 2024-04-02T04:57:54.839 INFO:teuthology.orchestra.run.smithi152.stdout:created 2024-04-02T04:55:50.446442+0000 2024-04-02T04:57:54.839 INFO:teuthology.orchestra.run.smithi152.stdout:modified 2024-04-02T04:57:49.433765+0000 2024-04-02T04:57:54.839 INFO:teuthology.orchestra.run.smithi152.stdout:tableserver 0 2024-04-02T04:57:54.839 INFO:teuthology.orchestra.run.smithi152.stdout:root 0 2024-04-02T04:57:54.839 INFO:teuthology.orchestra.run.smithi152.stdout:session_timeout 60 2024-04-02T04:57:54.839 INFO:teuthology.orchestra.run.smithi152.stdout:session_autoclose 300 2024-04-02T04:57:54.839 INFO:teuthology.orchestra.run.smithi152.stdout:max_file_size 1099511627776 2024-04-02T04:57:54.839 INFO:teuthology.orchestra.run.smithi152.stdout:required_client_features {} 2024-04-02T04:57:54.839 INFO:teuthology.orchestra.run.smithi152.stdout:last_failure 0 2024-04-02T04:57:54.839 INFO:teuthology.orchestra.run.smithi152.stdout:last_failure_osd_epoch 46 2024-04-02T04:57:54.840 INFO:teuthology.orchestra.run.smithi152.stdout:compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-02T04:57:54.840 INFO:teuthology.orchestra.run.smithi152.stdout:max_mds 2 2024-04-02T04:57:54.840 INFO:teuthology.orchestra.run.smithi152.stdout:in 0,1 2024-04-02T04:57:54.840 INFO:teuthology.orchestra.run.smithi152.stdout:up {0=14456,1=14470} 2024-04-02T04:57:54.840 INFO:teuthology.orchestra.run.smithi152.stdout:failed 2024-04-02T04:57:54.840 INFO:teuthology.orchestra.run.smithi152.stdout:damaged 2024-04-02T04:57:54.840 INFO:teuthology.orchestra.run.smithi152.stdout:stopped 2024-04-02T04:57:54.840 INFO:teuthology.orchestra.run.smithi152.stdout:data_pools [3] 2024-04-02T04:57:54.840 INFO:teuthology.orchestra.run.smithi152.stdout:metadata_pool 2 2024-04-02T04:57:54.840 INFO:teuthology.orchestra.run.smithi152.stdout:inline_data enabled 2024-04-02T04:57:54.840 INFO:teuthology.orchestra.run.smithi152.stdout:balancer 2024-04-02T04:57:54.840 INFO:teuthology.orchestra.run.smithi152.stdout:standby_count_wanted 1 2024-04-02T04:57:54.840 INFO:teuthology.orchestra.run.smithi152.stdout:[mds.cephfs.smithi152.lluqrs{0:14456} state up:active seq 5 export targets 1 join_fscid=1 addr [v2:172.21.15.152:6826/4038990242,v1:172.21.15.152:6827/4038990242] compat {c=[1],r=[1],i=[7ff]}] 2024-04-02T04:57:54.840 INFO:teuthology.orchestra.run.smithi152.stdout:[mds.cephfs.smithi154.rbkmwp{0:24297} state up:standby-replay seq 2 join_fscid=1 addr [v2:172.21.15.154:6826/53707828,v1:172.21.15.154:6827/53707828] compat {c=[1],r=[1],i=[7ff]}] 2024-04-02T04:57:54.840 INFO:teuthology.orchestra.run.smithi152.stdout:[mds.cephfs.smithi152.ywghzy{1:14470} state up:active seq 6 join_fscid=1 addr [v2:172.21.15.152:6828/3554217123,v1:172.21.15.152:6829/3554217123] compat {c=[1],r=[1],i=[7ff]}] 2024-04-02T04:57:54.841 INFO:teuthology.orchestra.run.smithi152.stdout:[mds.cephfs.smithi154.llvaoh{1:24301} state up:standby-replay seq 1 join_fscid=1 addr [v2:172.21.15.154:6824/1569612467,v1:172.21.15.154:6825/1569612467] compat {c=[1],r=[1],i=[7ff]}] 2024-04-02T04:57:54.841 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:57:54.841 INFO:teuthology.orchestra.run.smithi152.stdout: 2024-04-02T04:57:54.841 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.836+0000 7f085a920700 1 -- 172.21.15.152:0/1198141098 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f083c069300 msgr2=0x7f083c06b7b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:54.842 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.836+0000 7f085a920700 1 --2- 172.21.15.152:0/1198141098 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f083c069300 0x7f083c06b7b0 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 crypto rx=0x7f0848008fd0 tx=0x7f084800b040 comp rx=0 tx=0).stop 2024-04-02T04:57:54.842 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.836+0000 7f085a920700 1 -- 172.21.15.152:0/1198141098 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f08540fbce0 msgr2=0x7f0854110700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:54.842 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.836+0000 7f085a920700 1 --2- 172.21.15.152:0/1198141098 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f08540fbce0 0x7f0854110700 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 crypto rx=0x7f084400b0d0 tx=0x7f0844009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:54.842 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.836+0000 7f085a920700 1 -- 172.21.15.152:0/1198141098 shutdown_connections 2024-04-02T04:57:54.842 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.836+0000 7f085a920700 1 --2- 172.21.15.152:0/1198141098 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f083c069300 0x7f083c06b7b0 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:54.842 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.836+0000 7f085a920700 1 --2- 172.21.15.152:0/1198141098 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f08540fbce0 0x7f0854110700 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:54.842 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.836+0000 7f085a920700 1 --2- 172.21.15.152:0/1198141098 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f08540fcf90 0x7f0854112c50 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:54.842 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.836+0000 7f085a920700 1 -- 172.21.15.152:0/1198141098 >> 172.21.15.152:0/1198141098 conn(0x7f08540f7440 msgr2=0x7f08540f8b10 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:54.842 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.837+0000 7f085a920700 1 -- 172.21.15.152:0/1198141098 shutdown_connections 2024-04-02T04:57:54.842 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.837+0000 7f085a920700 1 -- 172.21.15.152:0/1198141098 wait complete. 2024-04-02T04:57:54.842 INFO:teuthology.orchestra.run.smithi152.stderr:dumped fsmap epoch 21 2024-04-02T04:57:54.987 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.983+0000 7f3e8e9f8700 1 -- 172.21.15.152:0/3621500082 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3e880fe440 msgr2=0x7f3e880fe850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:54.987 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.983+0000 7f3e8e9f8700 1 --2- 172.21.15.152:0/3621500082 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3e880fe440 0x7f3e880fe850 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7f3e7c004940 tx=0x7f3e7c009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:54.987 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.983+0000 7f3e8e9f8700 1 -- 172.21.15.152:0/3621500082 shutdown_connections 2024-04-02T04:57:54.987 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.983+0000 7f3e8e9f8700 1 --2- 172.21.15.152:0/3621500082 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3e880fed90 0x7f3e880faed0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:54.988 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.983+0000 7f3e8e9f8700 1 --2- 172.21.15.152:0/3621500082 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3e880fe440 0x7f3e880fe850 secure :-1 s=CLOSED pgs=309 cs=0 l=1 rev1=1 crypto rx=0x7f3e7c004940 tx=0x7f3e7c009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:54.988 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.983+0000 7f3e8e9f8700 1 -- 172.21.15.152:0/3621500082 >> 172.21.15.152:0/3621500082 conn(0x7f3e880f6bc0 msgr2=0x7f3e880f9010 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:54.988 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.983+0000 7f3e8e9f8700 1 -- 172.21.15.152:0/3621500082 shutdown_connections 2024-04-02T04:57:54.988 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.983+0000 7f3e8e9f8700 1 -- 172.21.15.152:0/3621500082 wait complete. 2024-04-02T04:57:54.989 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.984+0000 7f3e8e9f8700 1 Processor -- start 2024-04-02T04:57:54.989 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.984+0000 7f3e8e9f8700 1 -- start start 2024-04-02T04:57:54.989 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.984+0000 7f3e8e9f8700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3e880fed90 0x7f3e88107670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:54.989 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.985+0000 7f3e8e9f8700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3e88107bb0 0x7f3e88104110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:54.989 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.985+0000 7f3e8e9f8700 1 -- --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3e88108070 con 0x7f3e880fed90 2024-04-02T04:57:54.989 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.985+0000 7f3e8e9f8700 1 -- --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3e88104650 con 0x7f3e88107bb0 2024-04-02T04:57:54.989 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.985+0000 7f3e87fff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3e880fed90 0x7f3e88107670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:54.989 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.985+0000 7f3e877fe700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3e88107bb0 0x7f3e88104110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:54.989 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.985+0000 7f3e87fff700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3e880fed90 0x7f3e88107670 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:42860/0 (socket says 172.21.15.152:42860) 2024-04-02T04:57:54.990 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.985+0000 7f3e87fff700 1 -- 172.21.15.152:0/4198050126 learned_addr learned my addr 172.21.15.152:0/4198050126 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:57:54.995 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.986+0000 7f3e877fe700 1 -- 172.21.15.152:0/4198050126 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3e880fed90 msgr2=0x7f3e88107670 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:54.995 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.986+0000 7f3e877fe700 1 --2- 172.21.15.152:0/4198050126 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3e880fed90 0x7f3e88107670 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:54.995 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.986+0000 7f3e877fe700 1 -- 172.21.15.152:0/4198050126 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3e7c005040 con 0x7f3e88107bb0 2024-04-02T04:57:54.995 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.986+0000 7f3e877fe700 1 --2- 172.21.15.152:0/4198050126 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3e88107bb0 0x7f3e88104110 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f3e78009fd0 tx=0x7f3e7800b040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:54.996 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.986+0000 7f3e84ff9700 1 -- 172.21.15.152:0/4198050126 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f3e78004e90 con 0x7f3e88107bb0 2024-04-02T04:57:54.996 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.986+0000 7f3e84ff9700 1 -- 172.21.15.152:0/4198050126 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f3e7800f040 con 0x7f3e88107bb0 2024-04-02T04:57:54.996 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.987+0000 7f3e8e9f8700 1 -- 172.21.15.152:0/4198050126 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3e88104880 con 0x7f3e88107bb0 2024-04-02T04:57:54.996 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.987+0000 7f3e8e9f8700 1 -- 172.21.15.152:0/4198050126 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3e8810f950 con 0x7f3e88107bb0 2024-04-02T04:57:54.996 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.988+0000 7f3e84ff9700 1 -- 172.21.15.152:0/4198050126 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f3e7800ca50 con 0x7f3e88107bb0 2024-04-02T04:57:54.996 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.988+0000 7f3e8e9f8700 1 -- 172.21.15.152:0/4198050126 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3e740052e0 con 0x7f3e88107bb0 2024-04-02T04:57:54.996 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.990+0000 7f3e84ff9700 1 -- 172.21.15.152:0/4198050126 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f3e78007cc0 con 0x7f3e88107bb0 2024-04-02T04:57:54.996 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.991+0000 7f3e84ff9700 1 --2- 172.21.15.152:0/4198050126 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f3e70069090 0x7f3e7006b540 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:54.996 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.991+0000 7f3e84ff9700 1 -- 172.21.15.152:0/4198050126 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7f3e78087180 con 0x7f3e88107bb0 2024-04-02T04:57:54.996 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.991+0000 7f3e87fff700 1 --2- 172.21.15.152:0/4198050126 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f3e70069090 0x7f3e7006b540 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:54.997 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.992+0000 7f3e87fff700 1 --2- 172.21.15.152:0/4198050126 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f3e70069090 0x7f3e7006b540 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f3e7c0137b0 tx=0x7f3e7c009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:55.002 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:54.999+0000 7f3e84ff9700 1 -- 172.21.15.152:0/4198050126 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f3e78056f10 con 0x7f3e88107bb0 2024-04-02T04:57:55.236 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.232+0000 7f3e8e9f8700 1 -- 172.21.15.152:0/4198050126 --> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] -- mgr_command(tid 0: {"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}) v1 -- 0x7f3e74000bf0 con 0x7f3e70069090 2024-04-02T04:57:55.236 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.233+0000 7f3e84ff9700 1 -- 172.21.15.152:0/4198050126 <== mgr.14210 v2:172.21.15.152:6800/3117501261 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+328 (secure 0 0 0) 0x7f3e74000bf0 con 0x7f3e70069090 2024-04-02T04:57:55.237 INFO:teuthology.orchestra.run.smithi152.stdout:{ 2024-04-02T04:57:55.237 INFO:teuthology.orchestra.run.smithi152.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:7bd5b84926d4a0d6c51e5562e965bff211cec2740c9f4dec9b5e00a2bbb75eae", 2024-04-02T04:57:55.237 INFO:teuthology.orchestra.run.smithi152.stdout: "in_progress": true, 2024-04-02T04:57:55.237 INFO:teuthology.orchestra.run.smithi152.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-02T04:57:55.237 INFO:teuthology.orchestra.run.smithi152.stdout: "services_complete": [], 2024-04-02T04:57:55.237 INFO:teuthology.orchestra.run.smithi152.stdout: "progress": "", 2024-04-02T04:57:55.237 INFO:teuthology.orchestra.run.smithi152.stdout: "message": "Currently upgrading mgr daemons", 2024-04-02T04:57:55.237 INFO:teuthology.orchestra.run.smithi152.stdout: "is_paused": false 2024-04-02T04:57:55.237 INFO:teuthology.orchestra.run.smithi152.stdout:} 2024-04-02T04:57:55.239 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.235+0000 7f3e8e9f8700 1 -- 172.21.15.152:0/4198050126 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f3e70069090 msgr2=0x7f3e7006b540 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:55.239 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.235+0000 7f3e8e9f8700 1 --2- 172.21.15.152:0/4198050126 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f3e70069090 0x7f3e7006b540 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 crypto rx=0x7f3e7c0137b0 tx=0x7f3e7c009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:55.239 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.235+0000 7f3e8e9f8700 1 -- 172.21.15.152:0/4198050126 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3e88107bb0 msgr2=0x7f3e88104110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:55.239 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.235+0000 7f3e8e9f8700 1 --2- 172.21.15.152:0/4198050126 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3e88107bb0 0x7f3e88104110 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 crypto rx=0x7f3e78009fd0 tx=0x7f3e7800b040 comp rx=0 tx=0).stop 2024-04-02T04:57:55.239 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.235+0000 7f3e8e9f8700 1 -- 172.21.15.152:0/4198050126 shutdown_connections 2024-04-02T04:57:55.239 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.235+0000 7f3e8e9f8700 1 --2- 172.21.15.152:0/4198050126 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f3e70069090 0x7f3e7006b540 unknown :-1 s=CLOSED pgs=135 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:55.239 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.235+0000 7f3e8e9f8700 1 --2- 172.21.15.152:0/4198050126 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f3e880fed90 0x7f3e88107670 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:55.239 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.235+0000 7f3e8e9f8700 1 --2- 172.21.15.152:0/4198050126 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f3e88107bb0 0x7f3e88104110 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:55.239 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.235+0000 7f3e8e9f8700 1 -- 172.21.15.152:0/4198050126 >> 172.21.15.152:0/4198050126 conn(0x7f3e880f6bc0 msgr2=0x7f3e880f82f0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:55.239 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.235+0000 7f3e8e9f8700 1 -- 172.21.15.152:0/4198050126 shutdown_connections 2024-04-02T04:57:55.239 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.235+0000 7f3e8e9f8700 1 -- 172.21.15.152:0/4198050126 wait complete. 2024-04-02T04:57:55.350 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.346+0000 7f40e30f4700 1 -- 172.21.15.152:0/4205998703 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f40dc0fbd00 msgr2=0x7f40dc0fc110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:55.350 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.346+0000 7f40e30f4700 1 --2- 172.21.15.152:0/4205998703 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f40dc0fbd00 0x7f40dc0fc110 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 crypto rx=0x7f40d0004940 tx=0x7f40d0009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:55.350 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.346+0000 7f40e30f4700 1 -- 172.21.15.152:0/4205998703 shutdown_connections 2024-04-02T04:57:55.350 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.346+0000 7f40e30f4700 1 --2- 172.21.15.152:0/4205998703 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f40dc0fcfb0 0x7f40dc0fd420 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:55.350 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.346+0000 7f40e30f4700 1 --2- 172.21.15.152:0/4205998703 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f40dc0fbd00 0x7f40dc0fc110 unknown :-1 s=CLOSED pgs=310 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:55.350 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.346+0000 7f40e30f4700 1 -- 172.21.15.152:0/4205998703 >> 172.21.15.152:0/4205998703 conn(0x7f40dc0f7440 msgr2=0x7f40dc0f9890 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:55.350 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.346+0000 7f40e30f4700 1 -- 172.21.15.152:0/4205998703 shutdown_connections 2024-04-02T04:57:55.350 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.346+0000 7f40e30f4700 1 -- 172.21.15.152:0/4205998703 wait complete. 2024-04-02T04:57:55.352 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.347+0000 7f40e30f4700 1 Processor -- start 2024-04-02T04:57:55.352 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.347+0000 7f40e30f4700 1 -- start start 2024-04-02T04:57:55.352 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.347+0000 7f40e30f4700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f40dc0fbd00 0x7f40dc10c7b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:55.352 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.347+0000 7f40e0e90700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f40dc0fbd00 0x7f40dc10c7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:55.352 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.347+0000 7f40e0e90700 1 --2- >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f40dc0fbd00 0x7f40dc10c7b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_hello peer v2:172.21.15.152:3300/0 says I am v2:172.21.15.152:42872/0 (socket says 172.21.15.152:42872) 2024-04-02T04:57:55.352 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.347+0000 7f40e30f4700 1 --2- >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f40dc0fcfb0 0x7f40dc10ccf0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:55.352 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.348+0000 7f40e0e90700 1 -- 172.21.15.152:0/139464037 learned_addr learned my addr 172.21.15.152:0/139464037 (peer_addr_for_me v2:172.21.15.152:0/0) 2024-04-02T04:57:55.353 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.348+0000 7f40e30f4700 1 -- 172.21.15.152:0/139464037 --> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f40dc10d2f0 con 0x7f40dc0fbd00 2024-04-02T04:57:55.353 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.348+0000 7f40e30f4700 1 -- 172.21.15.152:0/139464037 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f40dc10d460 con 0x7f40dc0fcfb0 2024-04-02T04:57:55.353 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.348+0000 7f40dbfff700 1 --2- 172.21.15.152:0/139464037 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f40dc0fcfb0 0x7f40dc10ccf0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:55.353 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.348+0000 7f40dbfff700 1 -- 172.21.15.152:0/139464037 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f40dc0fbd00 msgr2=0x7f40dc10c7b0 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:55.353 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.348+0000 7f40dbfff700 1 --2- 172.21.15.152:0/139464037 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f40dc0fbd00 0x7f40dc10c7b0 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:55.353 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.348+0000 7f40dbfff700 1 -- 172.21.15.152:0/139464037 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f40d0005040 con 0x7f40dc0fcfb0 2024-04-02T04:57:55.353 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.348+0000 7f40e0e90700 1 --2- 172.21.15.152:0/139464037 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f40dc0fbd00 0x7f40dc10c7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).handle_auth_done state changed! 2024-04-02T04:57:55.353 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.348+0000 7f40dbfff700 1 --2- 172.21.15.152:0/139464037 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f40dc0fcfb0 0x7f40dc10ccf0 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f40c8009fd0 tx=0x7f40c800b040 comp rx=0 tx=0).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:55.354 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.349+0000 7f40d97fa700 1 -- 172.21.15.152:0/139464037 <== mon.1 v2:172.21.15.154:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f40c8004e90 con 0x7f40dc0fcfb0 2024-04-02T04:57:55.354 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.349+0000 7f40d97fa700 1 -- 172.21.15.152:0/139464037 <== mon.1 v2:172.21.15.154:3300/0 2 ==== config(27 keys) v1 ==== 1109+0+0 (secure 0 0 0) 0x7f40c800f040 con 0x7f40dc0fcfb0 2024-04-02T04:57:55.354 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.349+0000 7f40e30f4700 1 -- 172.21.15.152:0/139464037 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f40dc111e60 con 0x7f40dc0fcfb0 2024-04-02T04:57:55.354 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.349+0000 7f40d97fa700 1 -- 172.21.15.152:0/139464037 <== mon.1 v2:172.21.15.154:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f40c800ca50 con 0x7f40dc0fcfb0 2024-04-02T04:57:55.354 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.349+0000 7f40e30f4700 1 -- 172.21.15.152:0/139464037 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f40dc1123a0 con 0x7f40dc0fcfb0 2024-04-02T04:57:55.354 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.350+0000 7f40e30f4700 1 -- 172.21.15.152:0/139464037 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f40dc04c710 con 0x7f40dc0fcfb0 2024-04-02T04:57:55.354 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.351+0000 7f40d97fa700 1 -- 172.21.15.152:0/139464037 <== mon.1 v2:172.21.15.154:3300/0 4 ==== mgrmap(e 19) v1 ==== 86623+0+0 (secure 0 0 0) 0x7f40c800cc40 con 0x7f40dc0fcfb0 2024-04-02T04:57:55.355 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.351+0000 7f40d97fa700 1 --2- 172.21.15.152:0/139464037 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f40cc0692a0 0x7f40cc06b750 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0).connect 2024-04-02T04:57:55.355 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.351+0000 7f40d97fa700 1 -- 172.21.15.152:0/139464037 <== mon.1 v2:172.21.15.154:3300/0 5 ==== osd_map(46..46 src has 1..46) v4 ==== 5550+0+0 (secure 0 0 0) 0x7f40c8086cc0 con 0x7f40dc0fcfb0 2024-04-02T04:57:55.355 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.351+0000 7f40e0e90700 1 --2- 172.21.15.152:0/139464037 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f40cc0692a0 0x7f40cc06b750 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 crypto rx=0 tx=0 comp rx=0 tx=0)._handle_peer_banner_payload supported=3 required=0 2024-04-02T04:57:55.355 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.352+0000 7f40e0e90700 1 --2- 172.21.15.152:0/139464037 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f40cc0692a0 0x7f40cc06b750 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f40d0009fd0 tx=0x7f40d0009e30 comp rx=0 tx=0).ready entity=mgr.14210 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2024-04-02T04:57:55.363 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.359+0000 7f40d97fa700 1 -- 172.21.15.152:0/139464037 <== mon.1 v2:172.21.15.154:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+176504 (secure 0 0 0) 0x7f40c804e190 con 0x7f40dc0fcfb0 2024-04-02T04:57:55.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:55 smithi152 ceph-mon[29073]: from='client.14612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:57:55.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:55 smithi152 ceph-mon[29073]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:57:55.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:55 smithi152 ceph-mon[29073]: pgmap v236: 65 pgs: 65 active+clean; 68 MiB data, 2.0 GiB used, 534 GiB / 536 GiB avail; 2.0 MiB/s rd, 6.0 MiB/s wr, 514 op/s 2024-04-02T04:57:55.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:55 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/1198141098' entity='client.admin' cmd=[{"prefix": "fs dump"}]: dispatch 2024-04-02T04:57:55.507 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:55 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:57:55.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:55 smithi154 ceph-mon[35729]: from='client.14612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:57:55.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:55 smithi154 ceph-mon[35729]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:57:55.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:55 smithi154 ceph-mon[35729]: pgmap v236: 65 pgs: 65 active+clean; 68 MiB data, 2.0 GiB used, 534 GiB / 536 GiB avail; 2.0 MiB/s rd, 6.0 MiB/s wr, 514 op/s 2024-04-02T04:57:55.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:55 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/1198141098' entity='client.admin' cmd=[{"prefix": "fs dump"}]: dispatch 2024-04-02T04:57:55.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:55 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:57:55.739 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.736+0000 7f40e30f4700 1 -- 172.21.15.152:0/139464037 --> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] -- mon_command({"prefix": "health", "detail": "detail"} v 0) v1 -- 0x7f40dc1047f0 con 0x7f40dc0fcfb0 2024-04-02T04:57:55.740 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.736+0000 7f40d97fa700 1 -- 172.21.15.152:0/139464037 <== mon.1 v2:172.21.15.154:3300/0 7 ==== mon_command_ack([{"prefix": "health", "detail": "detail"}]=0 v0) v1 ==== 74+0+201 (secure 0 0 0) 0x7f40c8019090 con 0x7f40dc0fcfb0 2024-04-02T04:57:55.740 INFO:teuthology.orchestra.run.smithi152.stdout:HEALTH_WARN 1 filesystem with deprecated feature inline_data 2024-04-02T04:57:55.740 INFO:teuthology.orchestra.run.smithi152.stdout:[WRN] FS_INLINE_DATA_DEPRECATED: 1 filesystem with deprecated feature inline_data 2024-04-02T04:57:55.740 INFO:teuthology.orchestra.run.smithi152.stdout: fs cephfs has deprecated feature inline_data enabled. 2024-04-02T04:57:55.742 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.738+0000 7f40e30f4700 1 -- 172.21.15.152:0/139464037 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f40cc0692a0 msgr2=0x7f40cc06b750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:55.742 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.738+0000 7f40e30f4700 1 --2- 172.21.15.152:0/139464037 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f40cc0692a0 0x7f40cc06b750 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 crypto rx=0x7f40d0009fd0 tx=0x7f40d0009e30 comp rx=0 tx=0).stop 2024-04-02T04:57:55.742 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.738+0000 7f40e30f4700 1 -- 172.21.15.152:0/139464037 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f40dc0fcfb0 msgr2=0x7f40dc10ccf0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2024-04-02T04:57:55.742 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.738+0000 7f40e30f4700 1 --2- 172.21.15.152:0/139464037 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f40dc0fcfb0 0x7f40dc10ccf0 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 crypto rx=0x7f40c8009fd0 tx=0x7f40c800b040 comp rx=0 tx=0).stop 2024-04-02T04:57:55.743 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.738+0000 7f40e30f4700 1 -- 172.21.15.152:0/139464037 shutdown_connections 2024-04-02T04:57:55.743 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.738+0000 7f40e30f4700 1 --2- 172.21.15.152:0/139464037 >> [v2:172.21.15.152:6800/3117501261,v1:172.21.15.152:6801/3117501261] conn(0x7f40cc0692a0 0x7f40cc06b750 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:55.743 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.738+0000 7f40e30f4700 1 --2- 172.21.15.152:0/139464037 >> [v2:172.21.15.152:3300/0,v1:172.21.15.152:6789/0] conn(0x7f40dc0fbd00 0x7f40dc10c7b0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:55.743 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.738+0000 7f40e30f4700 1 --2- 172.21.15.152:0/139464037 >> [v2:172.21.15.154:3300/0,v1:172.21.15.154:6789/0] conn(0x7f40dc0fcfb0 0x7f40dc10ccf0 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 crypto rx=0 tx=0 comp rx=0 tx=0).stop 2024-04-02T04:57:55.743 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.739+0000 7f40e30f4700 1 -- 172.21.15.152:0/139464037 >> 172.21.15.152:0/139464037 conn(0x7f40dc0f7440 msgr2=0x7f40dc0f8aa0 unknown :-1 s=STATE_NONE l=0).mark_down 2024-04-02T04:57:55.743 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.739+0000 7f40e30f4700 1 -- 172.21.15.152:0/139464037 shutdown_connections 2024-04-02T04:57:55.743 INFO:teuthology.orchestra.run.smithi152.stderr:2024-04-02T04:57:55.739+0000 7f40e30f4700 1 -- 172.21.15.152:0/139464037 wait complete. 2024-04-02T04:57:56.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:56 smithi154 ceph-mon[35729]: from='client.24417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:57:56.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:56 smithi154 ceph-mon[35729]: from='client.? 172.21.15.152:0/139464037' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T04:57:56.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:56 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:57:56.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:56 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:57:56.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:56 smithi152 ceph-mon[29073]: from='client.24417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-02T04:57:56.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:56 smithi152 ceph-mon[29073]: from='client.? 172.21.15.152:0/139464037' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-02T04:57:56.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:56 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:57:56.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:56 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:57:57.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:57 smithi154 ceph-mon[35729]: pgmap v237: 65 pgs: 65 active+clean; 72 MiB data, 2.0 GiB used, 534 GiB / 536 GiB avail; 2.3 MiB/s rd, 6.1 MiB/s wr, 514 op/s 2024-04-02T04:57:57.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:57 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:57:57.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:57 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:57:57.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:57 smithi152 ceph-mon[29073]: pgmap v237: 65 pgs: 65 active+clean; 72 MiB data, 2.0 GiB used, 534 GiB / 536 GiB avail; 2.3 MiB/s rd, 6.1 MiB/s wr, 514 op/s 2024-04-02T04:57:57.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:57 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:57:57.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:57 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' 2024-04-02T04:57:59.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:59 smithi154 ceph-mon[35729]: pgmap v238: 65 pgs: 65 active+clean; 76 MiB data, 2.0 GiB used, 534 GiB / 536 GiB avail; 2.6 MiB/s rd, 6.4 MiB/s wr, 517 op/s 2024-04-02T04:57:59.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:59 smithi154 ceph-mon[35729]: Standby manager daemon smithi154.pplnxz restarted 2024-04-02T04:57:59.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:59 smithi154 ceph-mon[35729]: Standby manager daemon smithi154.pplnxz started 2024-04-02T04:57:59.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:59 smithi154 ceph-mon[35729]: from='mgr.? 172.21.15.154:0/2052763888' entity='mgr.smithi154.pplnxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi154.pplnxz/crt"}]: dispatch 2024-04-02T04:57:59.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:59 smithi154 ceph-mon[35729]: from='mgr.? 172.21.15.154:0/2052763888' entity='mgr.smithi154.pplnxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T04:57:59.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:59 smithi154 ceph-mon[35729]: from='mgr.? 172.21.15.154:0/2052763888' entity='mgr.smithi154.pplnxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi154.pplnxz/key"}]: dispatch 2024-04-02T04:57:59.734 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:57:59 smithi154 ceph-mon[35729]: from='mgr.? 172.21.15.154:0/2052763888' entity='mgr.smithi154.pplnxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T04:57:59.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:59 smithi152 ceph-mon[29073]: pgmap v238: 65 pgs: 65 active+clean; 76 MiB data, 2.0 GiB used, 534 GiB / 536 GiB avail; 2.6 MiB/s rd, 6.4 MiB/s wr, 517 op/s 2024-04-02T04:57:59.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:59 smithi152 ceph-mon[29073]: Standby manager daemon smithi154.pplnxz restarted 2024-04-02T04:57:59.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:59 smithi152 ceph-mon[29073]: Standby manager daemon smithi154.pplnxz started 2024-04-02T04:57:59.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:59 smithi152 ceph-mon[29073]: from='mgr.? 172.21.15.154:0/2052763888' entity='mgr.smithi154.pplnxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi154.pplnxz/crt"}]: dispatch 2024-04-02T04:57:59.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:59 smithi152 ceph-mon[29073]: from='mgr.? 172.21.15.154:0/2052763888' entity='mgr.smithi154.pplnxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-02T04:57:59.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:59 smithi152 ceph-mon[29073]: from='mgr.? 172.21.15.154:0/2052763888' entity='mgr.smithi154.pplnxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi154.pplnxz/key"}]: dispatch 2024-04-02T04:57:59.758 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:57:59 smithi152 ceph-mon[29073]: from='mgr.? 172.21.15.154:0/2052763888' entity='mgr.smithi154.pplnxz' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-02T04:58:00.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:58:00 smithi154 ceph-mon[35729]: mgrmap e20: smithi152.jivdcl(active, since 6m), standbys: smithi154.pplnxz 2024-04-02T04:58:00.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:58:00 smithi152 ceph-mon[29073]: mgrmap e20: smithi152.jivdcl(active, since 6m), standbys: smithi154.pplnxz 2024-04-02T04:58:01.756 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:58:01 smithi152 ceph-mon[29073]: pgmap v239: 65 pgs: 65 active+clean; 86 MiB data, 2.0 GiB used, 534 GiB / 536 GiB avail; 2.9 MiB/s rd, 6.8 MiB/s wr, 534 op/s 2024-04-02T04:58:01.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:58:01 smithi154 ceph-mon[35729]: pgmap v239: 65 pgs: 65 active+clean; 86 MiB data, 2.0 GiB used, 534 GiB / 536 GiB avail; 2.9 MiB/s rd, 6.8 MiB/s wr, 534 op/s 2024-04-02T04:58:03.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:58:03 smithi154 ceph-mon[35729]: pgmap v240: 65 pgs: 65 active+clean; 93 MiB data, 2.0 GiB used, 534 GiB / 536 GiB avail; 2.7 MiB/s rd, 5.4 MiB/s wr, 336 op/s 2024-04-02T04:58:03.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:58:03 smithi152 ceph-mon[29073]: pgmap v240: 65 pgs: 65 active+clean; 93 MiB data, 2.0 GiB used, 534 GiB / 536 GiB avail; 2.7 MiB/s rd, 5.4 MiB/s wr, 336 op/s 2024-04-02T04:58:05.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:58:05 smithi154 ceph-mon[35729]: pgmap v241: 65 pgs: 65 active+clean; 141 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail; 3.7 MiB/s rd, 8.8 MiB/s wr, 813 op/s 2024-04-02T04:58:05.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:58:05 smithi152 ceph-mon[29073]: pgmap v241: 65 pgs: 65 active+clean; 141 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail; 3.7 MiB/s rd, 8.8 MiB/s wr, 813 op/s 2024-04-02T04:58:07.733 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:58:07 smithi154 ceph-mon[35729]: pgmap v242: 65 pgs: 65 active+clean; 149 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail; 3.0 MiB/s rd, 7.1 MiB/s wr, 715 op/s 2024-04-02T04:58:07.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:58:07 smithi152 ceph-mon[29073]: pgmap v242: 65 pgs: 65 active+clean; 149 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail; 3.0 MiB/s rd, 7.1 MiB/s wr, 715 op/s 2024-04-02T04:58:08.286 INFO:tasks.workunit.client.0.smithi152.stderr:+ pushd ltp-full-20091231/testcases/kernel/fs/fsstress 2024-04-02T04:58:08.296 INFO:tasks.workunit.client.0.smithi152.stdout:~/cephtest/mnt.0/client.0/tmp/fsstress/ltp-full-20091231/testcases/kernel/fs/fsstress ~/cephtest/mnt.0/client.0/tmp/fsstress ~/cephtest/mnt.0/client.0/tmp 2024-04-02T04:58:08.296 INFO:tasks.workunit.client.0.smithi152.stderr:+ make 2024-04-02T04:58:08.312 INFO:tasks.workunit.client.1.smithi154.stderr:+ pushd ltp-full-20091231/testcases/kernel/fs/fsstress 2024-04-02T04:58:08.312 INFO:tasks.workunit.client.1.smithi154.stdout:~/cephtest/mnt.1/client.1/tmp/fsstress/ltp-full-20091231/testcases/kernel/fs/fsstress ~/cephtest/mnt.1/client.1/tmp/fsstress ~/cephtest/mnt.1/client.1/tmp 2024-04-02T04:58:08.312 INFO:tasks.workunit.client.1.smithi154.stderr:+ make 2024-04-02T04:58:08.398 INFO:tasks.workunit.client.0.smithi152.stdout:cc -DNO_XFS -I/home/ubuntu/cephtest/mnt.0/client.0/tmp/fsstress/ltp-full-20091231/testcases/kernel/fs/fsstress -D_LARGEFILE64_SOURCE -D_GNU_SOURCE -I../../../../include -I../../../../include -L../../../../lib fsstress.c -o fsstress 2024-04-02T04:58:08.764 INFO:tasks.workunit.client.1.smithi154.stdout:cc -DNO_XFS -I/home/ubuntu/cephtest/mnt.1/client.1/tmp/fsstress/ltp-full-20091231/testcases/kernel/fs/fsstress -D_LARGEFILE64_SOURCE -D_GNU_SOURCE -I../../../../include -I../../../../include -L../../../../lib fsstress.c -o fsstress 2024-04-02T04:58:09.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:58:09 smithi152 ceph-mon[29073]: pgmap v243: 65 pgs: 65 active+clean; 155 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 2.7 MiB/s rd, 7.4 MiB/s wr, 805 op/s 2024-04-02T04:58:09.983 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:58:09 smithi154 ceph-mon[35729]: pgmap v243: 65 pgs: 65 active+clean; 155 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 2.7 MiB/s rd, 7.4 MiB/s wr, 805 op/s 2024-04-02T04:58:10.693 INFO:journalctl@ceph.mon.smithi154.smithi154.stdout:Apr 02 04:58:10 smithi154 ceph-mon[35729]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T04:58:10.757 INFO:journalctl@ceph.mon.smithi152.smithi152.stdout:Apr 02 04:58:10 smithi152 ceph-mon[29073]: from='mgr.14210 172.21.15.152:0/2278315586' entity='mgr.smithi152.jivdcl' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-02T16:38:45.940 DEBUG:teuthology.exit:Got signal 15; running 1 handler... 2024-04-02T16:38:45.964 DEBUG:teuthology.task.console_log:Killing console logger for smithi152 2024-04-02T16:38:45.965 DEBUG:teuthology.task.console_log:Killing console logger for smithi154 2024-04-02T16:38:45.965 DEBUG:teuthology.exit:Finished running handlers